diff --git a/404.html b/404.html
index aab14683d1..c00af4a888 100644
--- a/404.html
+++ b/404.html
@@ -1,7 +1,7 @@
 <!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.101.0"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/favicons/android-96x96.png sizes=96x96><link rel=icon type=image/png href=/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/favicons/android-192x192.png sizes=192x192><title>404 Page not found | Cortex</title><meta name=description content><meta property="og:title" content="404 Page not found"><meta property="og:description" content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta property="og:type" content="website"><meta property="og:url" content="/404.html"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="404 Page not found"><meta itemprop=description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="404 Page not found"><meta name=twitter:description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-404><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/docs/><span>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-default td-outer"><main role=main class=td-main><main id=main><div><h1 id=title>Not found</h1><p>Oops! This page doesn't exist. Try going back to our <a href=/>home page</a>.</p><p>You can learn how to make a 404 page like this in <a href=https://gohugo.io/templates/404/>Custom 404 Pages</a>.</p></div></main></main><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-404><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/docs/><span>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-default td-outer"><main role=main class=td-main><main id=main><div><h1 id=title>Not found</h1><p>Oops! This page doesn't exist. Try going back to our <a href=/>home page</a>.</p><p>You can learn how to make a 404 page like this in <a href=https://gohugo.io/templates/404/>Custom 404 Pages</a>.</p></div></main></main><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
 For a list of trademarks of The Linux Foundation, please see <a class="text-light alert-link" href=https://www.linuxfoundation.org/trademark-usage>Trademark Usage page</a>.</small></div></footer></div><script src=https://cdn.jsdelivr.net/npm/popper.js@1.16.1/dist/umd/popper.min.js integrity=sha384-9/reFTGAW83EW2RDu2S0VKaIzap3H66lZH81PoYlFhbGU+6BZp6G7niu735Sk7lN crossorigin=anonymous></script>
 <script src=https://cdn.jsdelivr.net/npm/bootstrap@4.6.1/dist/js/bootstrap.min.js integrity="sha512-UR25UO94eTnCVwjbXozyeVd6ZqpaAE9naiEUBK/A+QDbfSTQFhPGj5lOR6d8tsgbBk84Ggb5A3EkjsOgPRPcKA==" crossorigin=anonymous></script>
 <script src=/js/tabpane-persist.js></script>
diff --git a/docs/api/index.html b/docs/api/index.html
index da786128e9..89546f0cf5 100644
--- a/docs/api/index.html
+++ b/docs/api/index.html
@@ -2,7 +2,7 @@
 For the sake of clarity, in this document we …"><meta property="og:title" content="HTTP API"><meta property="og:description" content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta property="og:type" content="website"><meta property="og:url" content="/docs/api/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="HTTP API"><meta itemprop=description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="HTTP API"><meta name=twitter:description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__section" id=m-docsapi><span class=td-sidebar-nav-active-item>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/api/_index.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=HTTP%20API" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#endpoints>Endpoints</a><ul><li><a href=#path-prefixes>Path prefixes</a></li><li><a href=#authentication>Authentication</a></li></ul></li><li><a href=#all-services>All services</a><ul><li><a href=#index-page>Index page</a></li><li><a href=#configuration>Configuration</a></li><li><a href=#runtime-configuration>Runtime Configuration</a></li><li><a href=#services-status>Services status</a></li><li><a href=#readiness-probe>Readiness probe</a></li><li><a href=#metrics>Metrics</a></li><li><a href=#pprof>Pprof</a></li><li><a href=#fgprof>Fgprof</a></li></ul></li><li><a href=#distributor>Distributor</a><ul><li><a href=#remote-write>Remote write</a></li><li><a href=#otlp-receiver>OTLP Receiver</a></li><li><a href=#distributor-ring-status>Distributor ring status</a></li><li><a href=#tenants-stats>Tenants stats</a></li><li><a href=#ha-tracker-status>HA tracker status</a></li></ul></li><li><a href=#ingester>Ingester</a><ul><li><a href=#flush-blocks>Flush blocks</a></li><li><a href=#shutdown>Shutdown</a></li><li><a href=#ingesters-ring-status>Ingesters ring status</a></li><li><a href=#ingester-tenants-stats>Ingester tenants stats</a></li><li><a href=#ingester-mode>Ingester mode</a></li></ul></li><li><a href=#querier--query-frontend>Querier / Query-frontend</a><ul><li><a href=#instant-query>Instant query</a></li><li><a href=#range-query>Range query</a></li><li><a href=#exemplar-query>Exemplar query</a></li><li><a href=#get-series-by-label-matchers>Get series by label matchers</a></li><li><a href=#get-label-names>Get label names</a></li><li><a href=#get-label-values>Get label values</a></li><li><a href=#get-metric-metadata>Get metric metadata</a></li><li><a href=#remote-read>Remote read</a></li><li><a href=#build-information>Build Information</a></li></ul></li><li><a href=#querier>Querier</a><ul><li><a href=#get-tenant-ingestion-stats>Get tenant ingestion stats</a></li></ul></li><li><a href=#ruler>Ruler</a><ul><li><a href=#ruler-ring-status>Ruler ring status</a></li><li><a href=#ruler-rules>Ruler rules</a></li><li><a href=#list-rules>List rules</a></li><li><a href=#list-alerts>List alerts</a></li><li><a href=#list-rule-groups>List rule groups</a></li><li><a href=#get-rule-groups-by-namespace>Get rule groups by namespace</a></li><li><a href=#get-rule-group>Get rule group</a></li><li><a href=#set-rule-group>Set rule group</a></li><li><a href=#delete-rule-group>Delete rule group</a></li><li><a href=#delete-namespace>Delete namespace</a></li><li><a href=#delete-tenant-configuration>Delete tenant configuration</a></li></ul></li><li><a href=#alertmanager>Alertmanager</a><ul><li><a href=#alertmanager-status>Alertmanager status</a></li><li><a href=#alertmanager-configs>Alertmanager configs</a></li><li><a href=#alertmanager-ring-status>Alertmanager ring status</a></li><li><a href=#alertmanager-ui>Alertmanager UI</a></li><li><a href=#alertmanager-delete-tenant-configuration>Alertmanager Delete Tenant Configuration</a></li><li><a href=#get-alertmanager-configuration>Get Alertmanager configuration</a></li><li><a href=#set-alertmanager-configuration>Set Alertmanager configuration</a></li><li><a href=#delete-alertmanager-configuration>Delete Alertmanager configuration</a></li></ul></li><li><a href=#purger>Purger</a><ul><li><a href=#tenant-delete-request>Tenant Delete Request</a></li><li><a href=#tenant-delete-status>Tenant Delete Status</a></li></ul></li><li><a href=#store-gateway>Store-gateway</a><ul><li><a href=#store-gateway-ring-status>Store-gateway ring status</a></li></ul></li><li><a href=#compactor>Compactor</a><ul><li><a href=#compactor-ring-status>Compactor ring status</a></li></ul></li><li><a href=#configs-api>Configs API</a><ul><li></li><li><a href=#get-rule-files>Get rule files</a></li><li><a href=#set-rule-files>Set rule files</a></li><li><a href=#get-template-files>Get template files</a></li><li><a href=#set-template-files>Set template files</a></li><li><a href=#set-alertmanager-config-file>Set Alertmanager config file</a></li><li><a href=#validate-alertmanager-config-file>Validate Alertmanager config file</a></li><li><a href=#deactivate-configs>Deactivate configs</a></li><li><a href=#restore-configs>Restore configs</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/api/ aria-disabled=true class="btn-link disabled">HTTP API</a></li></ol></nav><div class=td-content><h1>HTTP API</h1><header class=article-meta></header><p>Cortex exposes an HTTP API for pushing and querying time series data, and operating the cluster itself.</p><p>For the sake of clarity, in this document we have grouped API endpoints by service, but keep in mind that they&rsquo;re exposed both when running Cortex in microservices and singly-binary mode:</p><ul><li><strong>Microservices</strong>: each service exposes its own endpoints</li><li><strong>Single-binary</strong>: the Cortex process exposes all API endpoints for the services running internally</li></ul><h2 id=endpoints>Endpoints</h2><table><thead><tr><th>API</th><th>Service</th><th>Available Since</th><th>Endpoint</th></tr></thead><tbody><tr><td><a href=#index-page>Index page</a></td><td><em>All services</em></td><td></td><td><code>GET /</code></td></tr><tr><td><a href=#configuration>Configuration</a></td><td><em>All services</em></td><td></td><td><code>GET /config</code></td></tr><tr><td><a href=#runtime-configuration>Runtime Configuration</a></td><td><em>All services</em></td><td></td><td><code>GET /runtime_config</code></td></tr><tr><td><a href=#services-status>Services status</a></td><td><em>All services</em></td><td></td><td><code>GET /services</code></td></tr><tr><td><a href=#readiness-probe>Readiness probe</a></td><td><em>All services</em></td><td></td><td><code>GET /ready</code></td></tr><tr><td><a href=#metrics>Metrics</a></td><td><em>All services</em></td><td></td><td><code>GET /metrics</code></td></tr><tr><td><a href=#pprof>Pprof</a></td><td><em>All services</em></td><td></td><td><code>GET /debug/pprof</code></td></tr><tr><td><a href=#fgprof>Fgprof</a></td><td><em>All services</em></td><td></td><td><code>GET /debug/fgprof</code></td></tr><tr><td><a href=#remote-write>Remote write</a></td><td>Distributor</td><td></td><td><code>POST /api/v1/push</code></td></tr><tr><td><a href=#otlp-receiver>OTLP receiver</a></td><td>Distributor</td><td></td><td><code>POST /api/v1/otlp/v1/metrics</code></td></tr><tr><td><a href=#tenants-stats>Tenants stats</a></td><td>Distributor</td><td></td><td><code>GET /distributor/all_user_stats</code></td></tr><tr><td><a href=#ha-tracker-status>HA tracker status</a></td><td>Distributor</td><td></td><td><code>GET /distributor/ha_tracker</code></td></tr><tr><td><a href=#flush-blocks>Flush blocks</a></td><td>Ingester</td><td></td><td><code>GET,POST /ingester/flush</code></td></tr><tr><td><a href=#shutdown>Shutdown</a></td><td>Ingester</td><td></td><td><code>GET,POST /ingester/shutdown</code></td></tr><tr><td><a href=#ingesters-ring-status>Ingesters ring status</a></td><td>Ingester</td><td></td><td><code>GET /ingester/ring</code></td></tr><tr><td><a href=#ingester-tenants-stats>Ingester tenants stats</a></td><td>Ingester</td><td></td><td><code>GET /ingester/all_user_stats</code></td></tr><tr><td><a href=#ingester-mode>Ingester mode</a></td><td>Ingester</td><td></td><td><code>GET,POST /ingester/mode</code></td></tr><tr><td><a href=#instant-query>Instant query</a></td><td>Querier, Query-frontend</td><td></td><td><code>GET,POST &lt;prometheus-http-prefix>/api/v1/query</code></td></tr><tr><td><a href=#range-query>Range query</a></td><td>Querier, Query-frontend</td><td></td><td><code>GET,POST &lt;prometheus-http-prefix>/api/v1/query_range</code></td></tr><tr><td><a href=#exemplar-query>Exemplar query</a></td><td>Querier, Query-frontend</td><td></td><td><code>GET,POST &lt;prometheus-http-prefix>/api/v1/query_exemplars</code></td></tr><tr><td><a href=#get-series-by-label-matchers>Get series by label matchers</a></td><td>Querier, Query-frontend</td><td></td><td><code>GET,POST &lt;prometheus-http-prefix>/api/v1/series</code></td></tr><tr><td><a href=#get-label-names>Get label names</a></td><td>Querier, Query-frontend</td><td></td><td><code>GET,POST &lt;prometheus-http-prefix>/api/v1/labels</code></td></tr><tr><td><a href=#get-label-values>Get label values</a></td><td>Querier, Query-frontend</td><td></td><td><code>GET &lt;prometheus-http-prefix>/api/v1/label/{name}/values</code></td></tr><tr><td><a href=#get-metric-metadata>Get metric metadata</a></td><td>Querier, Query-frontend</td><td></td><td><code>GET &lt;prometheus-http-prefix>/api/v1/metadata</code></td></tr><tr><td><a href=#remote-read>Remote read</a></td><td>Querier, Query-frontend</td><td></td><td><code>POST &lt;prometheus-http-prefix>/api/v1/read</code></td></tr><tr><td><a href=#build-information>Build information</a></td><td>Querier, Query-frontend</td><td>v1.15.0</td><td><code>GET &lt;prometheus-http-prefix>/api/v1/status/buildinfo</code></td></tr><tr><td><a href=#get-tenant-ingestion-stats>Get tenant ingestion stats</a></td><td>Querier</td><td></td><td><code>GET /api/v1/user_stats</code></td></tr><tr><td><a href=#ruler-ring-status>Ruler ring status</a></td><td>Ruler</td><td></td><td><code>GET /ruler/ring</code></td></tr><tr><td><a href=#ruler-rule-groups>Ruler rules</a></td><td>Ruler</td><td></td><td><code>GET /ruler/rule_groups</code></td></tr><tr><td><a href=#list-rules>List rules</a></td><td>Ruler</td><td></td><td><code>GET &lt;prometheus-http-prefix>/api/v1/rules</code></td></tr><tr><td><a href=#list-alerts>List alerts</a></td><td>Ruler</td><td></td><td><code>GET &lt;prometheus-http-prefix>/api/v1/alerts</code></td></tr><tr><td><a href=#list-rule-groups>List rule groups</a></td><td>Ruler</td><td></td><td><code>GET /api/v1/rules</code></td></tr><tr><td><a href=#get-rule-groups-by-namespace>Get rule groups by namespace</a></td><td>Ruler</td><td></td><td><code>GET /api/v1/rules/{namespace}</code></td></tr><tr><td><a href=#get-rule-group>Get rule group</a></td><td>Ruler</td><td></td><td><code>GET /api/v1/rules/{namespace}/{groupName}</code></td></tr><tr><td><a href=#set-rule-group>Set rule group</a></td><td>Ruler</td><td></td><td><code>POST /api/v1/rules/{namespace}</code></td></tr><tr><td><a href=#delete-rule-group>Delete rule group</a></td><td>Ruler</td><td></td><td><code>DELETE /api/v1/rules/{namespace}/{groupName}</code></td></tr><tr><td><a href=#delete-namespace>Delete namespace</a></td><td>Ruler</td><td></td><td><code>DELETE /api/v1/rules/{namespace}</code></td></tr><tr><td><a href=#delete-tenant-configuration>Delete tenant configuration</a></td><td>Ruler</td><td></td><td><code>POST /ruler/delete_tenant_config</code></td></tr><tr><td><a href=#alertmanager-status>Alertmanager status</a></td><td>Alertmanager</td><td></td><td><code>GET /multitenant_alertmanager/status</code></td></tr><tr><td><a href=#alertmanager-configs>Alertmanager configs</a></td><td>Alertmanager</td><td></td><td><code>GET /multitenant_alertmanager/configs</code></td></tr><tr><td><a href=#alertmanager-ring-status>Alertmanager ring status</a></td><td>Alertmanager</td><td></td><td><code>GET /multitenant_alertmanager/ring</code></td></tr><tr><td><a href=#alertmanager-ui>Alertmanager UI</a></td><td>Alertmanager</td><td></td><td><code>GET /&lt;alertmanager-http-prefix></code></td></tr><tr><td><a href=#alertmanager-delete-tenant-configuration>Alertmanager Delete Tenant Configuration</a></td><td>Alertmanager</td><td></td><td><code>POST /multitenant_alertmanager/delete_tenant_config</code></td></tr><tr><td><a href=#get-alertmanager-configuration>Get Alertmanager configuration</a></td><td>Alertmanager</td><td></td><td><code>GET /api/v1/alerts</code></td></tr><tr><td><a href=#set-alertmanager-configuration>Set Alertmanager configuration</a></td><td>Alertmanager</td><td></td><td><code>POST /api/v1/alerts</code></td></tr><tr><td><a href=#delete-alertmanager-configuration>Delete Alertmanager configuration</a></td><td>Alertmanager</td><td></td><td><code>DELETE /api/v1/alerts</code></td></tr><tr><td><a href=#tenant-delete-request>Tenant delete request</a></td><td>Purger</td><td></td><td><code>POST /purger/delete_tenant</code></td></tr><tr><td><a href=#tenant-delete-status>Tenant delete status</a></td><td>Purger</td><td></td><td><code>GET /purger/delete_tenant_status</code></td></tr><tr><td><a href=#store-gateway-ring-status>Store-gateway ring status</a></td><td>Store-gateway</td><td></td><td><code>GET /store-gateway/ring</code></td></tr><tr><td><a href=#compactor-ring-status>Compactor ring status</a></td><td>Compactor</td><td></td><td><code>GET /compactor/ring</code></td></tr><tr><td><a href=#get-rule-files>Get rule files</a></td><td>Configs API (deprecated)</td><td></td><td><code>GET /api/prom/configs/rules</code></td></tr><tr><td><a href=#set-rule-files>Set rule files</a></td><td>Configs API (deprecated)</td><td></td><td><code>POST /api/prom/configs/rules</code></td></tr><tr><td><a href=#get-template-files>Get template files</a></td><td>Configs API (deprecated)</td><td></td><td><code>GET /api/prom/configs/templates</code></td></tr><tr><td><a href=#set-template-files>Set template files</a></td><td>Configs API (deprecated)</td><td></td><td><code>POST /api/prom/configs/templates</code></td></tr><tr><td><a href=#get-alertmanager-config-file>Get Alertmanager config file</a></td><td>Configs API (deprecated)</td><td></td><td><code>GET /api/prom/configs/alertmanager</code></td></tr><tr><td><a href=#set-alertmanager-config-file>Set Alertmanager config file</a></td><td>Configs API (deprecated)</td><td></td><td><code>POST /api/prom/configs/alertmanager</code></td></tr><tr><td><a href=#validate-alertmanager-config-file>Validate Alertmanager config</a></td><td>Configs API (deprecated)</td><td></td><td><code>POST /api/prom/configs/alertmanager/validate</code></td></tr><tr><td><a href=#deactivate-configs>Deactivate configs</a></td><td>Configs API (deprecated)</td><td></td><td><code>DELETE /api/prom/configs/deactivate</code></td></tr><tr><td><a href=#restore-configs>Restore configs</a></td><td>Configs API (deprecated)</td><td></td><td><code>POST /api/prom/configs/restore</code></td></tr></tbody></table><h3 id=path-prefixes>Path prefixes</h3><p>In this documentation you will find the usage of some placeholders for the path prefixes, whenever the prefix is configurable. The following table shows the supported prefixes.</p><table><thead><tr><th>Prefix</th><th>Default</th><th>CLI Flag</th><th>YAML Config</th></tr></thead><tbody><tr><td><code>&lt;legacy-http-prefix></code></td><td><code>/api/prom</code></td><td><code>-http.prefix</code></td><td><code>http_prefix</code></td></tr><tr><td><code>&lt;prometheus-http-prefix></code></td><td><code>/prometheus</code></td><td><code>-http.prometheus-http-prefix</code></td><td><code>api > prometheus_http_prefix</code></td></tr><tr><td><code>&lt;alertmanager-http-prefix></code></td><td><code>/alertmanager</code></td><td><code>-http.alertmanager-http-prefix</code></td><td><code>api > alertmanager_http_prefix</code></td></tr></tbody></table><h3 id=authentication>Authentication</h3><p>When multi-tenancy is enabled, endpoints requiring authentication are expected to be called with the <code>X-Scope-OrgID</code> HTTP request header set to the tenant ID. Otherwise, when multi-tenancy is disabled, Cortex doesn&rsquo;t require any request to have the <code>X-Scope-OrgID</code> header.</p><p>Multi-tenancy can be enabled/disabled via the CLI flag <code>-auth.enabled</code> or its respective YAML config option.</p><p><em>For more information, please refer to the dedicated <a href=/docs/guides/auth/>Authentication and Authorisation</a> guide.</em></p><h2 id=all-services>All services</h2><p>The following API endpoints are exposed by all services.</p><h3 id=index-page>Index page</h3><pre tabindex=0><code>GET /
diff --git a/docs/architecture/index.html b/docs/architecture/index.html
index 35f9997908..5eae9d82cd 100644
--- a/docs/architecture/index.html
+++ b/docs/architecture/index.html
@@ -7,7 +7,7 @@
 The role of Prometheus Prometheus instances scrape samples from various targets and then push them to Cortex (using Prometheus&rsquo; remote write API)."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span class=td-sidebar-nav-active-item>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/architecture.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Cortex%20Architecture" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#the-role-of-prometheus>The role of Prometheus</a></li><li><a href=#blocks-storage>Blocks storage</a></li><li><a href=#services>Services</a><ul><li><a href=#distributor>Distributor</a></li><li><a href=#ingester>Ingester</a></li><li><a href=#querier>Querier</a></li><li><a href=#compactor>Compactor</a></li><li><a href=#store-gateway>Store gateway</a></li><li><a href=#query-frontend>Query frontend</a></li><li><a href=#query-scheduler>Query Scheduler</a></li><li><a href=#ruler>Ruler</a></li><li><a href=#alertmanager>Alertmanager</a></li><li><a href=#configs-api>Configs API</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/architecture/ aria-disabled=true class="btn-link disabled">Architecture</a></li></ol></nav><div class=td-content><h1>Cortex Architecture</h1><header class=article-meta></header><p>Cortex consists of multiple horizontally scalable microservices. Each microservice uses the most appropriate technique for horizontal scaling; most are stateless and can handle requests for any users while some (namely the <a href=#ingester>ingesters</a>) are semi-stateful and depend on consistent hashing. This document provides a basic overview of Cortex&rsquo;s architecture.</p><p>The following diagram does not include all the Cortex services, but does represent a typical deployment topology.</p><p align=center><img src=/images/architecture.png alt="Cortex Architecture"></p><h2 id=the-role-of-prometheus>The role of Prometheus</h2><p>Prometheus instances scrape samples from various targets and then push them to Cortex (using Prometheus&rsquo; <a href=https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations>remote write API</a>). That remote write API emits batched <a href=https://google.github.io/snappy/>Snappy</a>-compressed <a href=https://developers.google.com/protocol-buffers/>Protocol Buffer</a> messages inside the body of an HTTP <code>PUT</code> request.</p><p>Cortex requires that each HTTP request bear a header specifying a tenant ID for the request. Request authentication and authorization are handled by an external reverse proxy.</p><p>Incoming samples (writes from Prometheus) are handled by the <a href=#distributor>distributor</a> while incoming reads (PromQL queries) are handled by the <a href=#querier>querier</a> or optionally by the <a href=#query-frontend>query frontend</a>.</p><h2 id=blocks-storage>Blocks storage</h2><p>The blocks storage is based on <a href=https://prometheus.io/docs/prometheus/latest/storage/>Prometheus TSDB</a>: it stores each tenant&rsquo;s time series into their own TSDB which write out their series to a on-disk Block (defaults to 2h block range periods). Each Block is composed by a few files storing the chunks and the block index.</p><p>The TSDB chunk files contain the samples for multiple series. The series inside the Chunks are then indexed by a per-block index, which indexes metric names and labels to time series in the chunk files.</p><p>The blocks storage doesn&rsquo;t require a dedicated storage backend for the index. The only requirement is an object store for the Block files, which can be:</p><ul><li><a href=https://aws.amazon.com/s3>Amazon S3</a></li><li><a href=https://cloud.google.com/storage/>Google Cloud Storage</a></li><li><a href=https://azure.microsoft.com/en-us/services/storage/>Microsoft Azure Storage</a></li><li><a href=https://wiki.openstack.org/wiki/Swift>OpenStack Swift</a></li><li><a href=https://thanos.io/storage.md/#filesystem>Local Filesystem</a> (single node only)</li></ul><p>For more information, please check out the <a href=/docs/blocks-storage/>Blocks storage</a> documentation.</p><h2 id=services>Services</h2><p>Cortex has a service-based architecture, in which the overall system is split up into a variety of components that perform a specific task. These components run separately and in parallel. Cortex can alternatively run in a single process mode, where all components are executed within a single process. The single process mode is particularly handy for local testing and development.</p><p>The Cortex services are:</p><ul><li><a href=#distributor>Distributor</a></li><li><a href=#ingester>Ingester</a></li><li><a href=#querier>Querier</a></li><li><a href=#compactor>Compactor</a></li><li><a href=#store-gateway>Store gateway</a></li><li><a href=#alertmanager>Alertmanager</a> (optional)</li><li><a href=#configs-api>Configs API</a> (optional)</li><li><a href=/docs/guides/overrides-exporter/>Overrides exporter</a> (optional)</li><li><a href=#query-frontend>Query frontend</a> (optional)</li><li><a href=/docs/operations/scaling-query-frontend/#query-scheduler>Query scheduler</a> (optional)</li><li><a href=#ruler>Ruler</a> (optional)</li></ul><h3 id=distributor>Distributor</h3><p>The <strong>distributor</strong> service is responsible for handling incoming samples from Prometheus. It&rsquo;s the first stop in the write path for series samples. Once the distributor receives samples from Prometheus, each sample is validated for correctness and to ensure that it is within the configured tenant limits, falling back to default ones in case limits have not been overridden for the specific tenant. Valid samples are then split into batches and sent to multiple <a href=#ingester>ingesters</a> in parallel.</p><p>The validation done by the distributor includes:</p><ul><li>The metric labels name are formally correct</li><li>The configured max number of labels per metric is respected</li><li>The configured max length of a label name and value is respected</li><li>The timestamp is not older/newer than the configured min/max time range</li></ul><p>Distributors are <strong>stateless</strong> and can be scaled up and down as needed.</p><h4 id=high-availability-tracker>High Availability Tracker</h4><p>The distributor features a <strong>High Availability (HA) Tracker</strong>. When enabled, the distributor deduplicates incoming samples from redundant Prometheus servers. This allows you to have multiple HA replicas of the same Prometheus servers, writing the same series to Cortex and then deduplicate these series in the Cortex distributor.</p><p>The HA Tracker deduplicates incoming samples based on a cluster and replica label. The cluster label uniquely identifies the cluster of redundant Prometheus servers for a given tenant, while the replica label uniquely identifies the replica within the Prometheus cluster. Incoming samples are considered duplicated (and thus dropped) if received by any replica which is not the current primary within a cluster.</p><p>The HA Tracker requires a key-value (KV) store to coordinate which replica is currently elected. The distributor will only accept samples from the current leader. Samples with one or no labels (of the replica and cluster) are accepted by default and never deduplicated.</p><p>The supported KV stores for the HA tracker are:</p><ul><li><a href=https://www.consul.io>Consul</a></li><li><a href=https://etcd.io>Etcd</a></li></ul><p>Note: Memberlist is not supported. Memberlist-based KV store propagates updates using gossip, which is very slow for HA purposes: result is that different distributors may see different Prometheus server as elected HA replica, which is definitely not desirable.</p><p>For more information, please refer to <a href=/docs/guides/ha-pair-handling/>config for sending HA pairs data to Cortex</a> in the documentation.</p><h4 id=hashing>Hashing</h4><p>Distributors use consistent hashing, in conjunction with a configurable replication factor, to determine which ingester instance(s) should receive a given series.</p><p>Cortex supports two hashing strategies:</p><ol><li>Hash the metric name and tenant ID (default)</li><li>Hash the metric name, labels and tenant ID (enabled with <code>-distributor.shard-by-all-labels=true</code>)</li></ol><p>The trade-off associated with the latter is that writes are more balanced across ingesters but each query needs to talk to all ingesters since a metric could be spread across multiple ingesters given different label sets.</p><h4 id=the-hash-ring>The hash ring</h4><p>A hash ring (stored in a key-value store) is used to achieve consistent hashing for the series sharding and replication across the ingesters. All <a href=#ingester>ingesters</a> register themselves into the hash ring with a set of tokens they own; each token is a random unsigned 32-bit number. Each incoming series is <a href=#hashing>hashed</a> in the distributor and then pushed to the ingester owning the tokens range for the series hash number plus N-1 subsequent ingesters in the ring, where N is the replication factor.</p><p>To do the hash lookup, distributors find the smallest appropriate token whose value is larger than the <a href=#hashing>hash of the series</a>. When the replication factor is larger than 1, the next subsequent tokens (clockwise in the ring) that belong to different ingesters will also be included in the result.</p><p>The effect of this hash set up is that each token that an ingester owns is responsible for a range of hashes. If there are three tokens with values 0, 25, and 50, then a hash of 3 would be given to the ingester that owns the token 25; the ingester owning token 25 is responsible for the hash range of 1-25.</p><p>The supported KV stores for the hash ring are:</p><ul><li><a href=https://www.consul.io>Consul</a></li><li><a href=https://etcd.io>Etcd</a></li><li>Gossip <a href=https://github.com/hashicorp/memberlist>memberlist</a></li></ul><h4 id=quorum-consistency>Quorum consistency</h4><p>Since all distributors share access to the same hash ring, write requests can be sent to any distributor and you can setup a stateless load balancer in front of it.</p><p>To ensure consistent query results, Cortex uses <a href=https://www.allthingsdistributed.com/files/amazon-dynamo-sosp2007.pdf>Dynamo-style</a> quorum consistency on reads and writes. This means that the distributor will wait for a positive response of at least one half plus one of the ingesters to send the sample to before successfully responding to the Prometheus write request.</p><h4 id=load-balancing-across-distributors>Load balancing across distributors</h4><p>We recommend randomly load balancing write requests across distributor instances. For example, if you&rsquo;re running Cortex in a Kubernetes cluster, you could run the distributors as a Kubernetes <a href=https://kubernetes.io/docs/concepts/services-networking/service/>Service</a>.</p><h3 id=ingester>Ingester</h3><p>The <strong>ingester</strong> service is responsible for writing incoming series to a <a href=#blocks-storage>long-term storage backend</a> on the write path and returning in-memory series samples for queries on the read path.</p><p>Incoming series are not immediately written to the storage but kept in memory and periodically flushed to the storage (by default, 2 hours). For this reason, the <a href=#querier>queriers</a> may need to fetch samples both from ingesters and long-term storage while executing a query on the read path.</p><p>Ingesters contain a <strong>lifecycler</strong> which manages the lifecycle of an ingester and stores the <strong>ingester state</strong> in the <a href=#the-hash-ring>hash ring</a>. Each ingester could be in one of the following states:</p><ul><li><strong><code>PENDING</code></strong><br>The ingester has just started. While in this state, the ingester doesn&rsquo;t receive neither write and read requests.</li><li><strong><code>JOINING</code></strong><br>The ingester is starting up and joining the ring. While in this state the ingester doesn&rsquo;t receive neither write and read requests. The ingester will join the ring using tokens loaded from disk (if <code>-ingester.tokens-file-path</code> is configured) or generate a set of new random ones. Finally, the ingester optionally observes the ring for tokens conflicts and then, once any conflict is resolved, will move to <code>ACTIVE</code> state.</li><li><strong><code>ACTIVE</code></strong><br>The ingester is up and running. While in this state the ingester can receive both write and read requests.</li><li><strong><code>LEAVING</code></strong><br>The ingester is shutting down and leaving the ring. While in this state the ingester doesn&rsquo;t receive write requests, while it could receive read requests.</li><li><strong><code>UNHEALTHY</code></strong><br>The ingester has failed to heartbeat to the ring&rsquo;s KV Store. While in this state, distributors skip the ingester while building the replication set for incoming series and the ingester does not receive write or read requests.</li></ul><p>Ingesters are <strong>semi-stateful</strong>.</p><h4 id=ingesters-failure-and-data-loss>Ingesters failure and data loss</h4><p>If an ingester process crashes or exits abruptly, all the in-memory series that have not yet been flushed to the long-term storage will be lost. There are two main ways to mitigate this failure mode:</p><ol><li>Replication</li><li>Write-ahead log (WAL)</li></ol><p>The <strong>replication</strong> is used to hold multiple (typically 3) replicas of each time series in the ingesters. If the Cortex cluster loses an ingester, the in-memory series held by the lost ingester are also replicated to at least another ingester. In the event of a single ingester failure, no time series samples will be lost. However, in the event of multiple ingester failures, time series may be potentially lost if the failures affect all the ingesters holding the replicas of a specific time series.</p><p>The <strong>write-ahead log</strong> (WAL) is used to write to a persistent disk all incoming series samples until they&rsquo;re flushed to the long-term storage. In the event of an ingester failure, a subsequent process restart will replay the WAL and recover the in-memory series samples.</p><p>Contrary to the sole replication and given the persistent disk data is not lost, in the event of multiple ingesters failure each ingester will recover the in-memory series samples from WAL upon subsequent restart. The replication is still recommended in order to ensure no temporary failures on the read path in the event of a single ingester failure.</p><h4 id=ingesters-write-de-amplification>Ingesters write de-amplification</h4><p>Ingesters store recently received samples in-memory in order to perform write de-amplification. If the ingesters would immediately write received samples to the long-term storage, the system would be very difficult to scale due to the very high pressure on the storage. For this reason, the ingesters batch and compress samples in-memory and periodically flush them out to the storage.</p><p>Write de-amplification is the main source of Cortex&rsquo;s low total cost of ownership (TCO).</p><h3 id=querier>Querier</h3><p>The <strong>querier</strong> service handles queries using the <a href=https://prometheus.io/docs/prometheus/latest/querying/basics/>PromQL</a> query language.</p><p>Queriers fetch series samples both from the ingesters and long-term storage. The ingesters hold the in-memory series that have not yet been flushed to the long-term storage. Because of the replication factor, it is possible that the querier may receive duplicated samples. To address this, the querier internally <strong>deduplicates</strong> samples with the exact same timestamp for a given time series.</p><p>Queriers are <strong>stateless</strong> and can be scaled up and down as needed.</p><h3 id=compactor>Compactor</h3><p>The <strong>compactor</strong> is a service which is responsible to:</p><ul><li>Compact multiple blocks of a given tenant into a single optimized larger block. This helps to reduce storage costs (deduplication, index size reduction), and increase query speed (querying fewer blocks is faster).</li><li>Keep the per-tenant bucket index updated. The <a href=/docs/blocks-storage/bucket-index/>bucket index</a> is used by <a href=/docs/blocks-storage/querier/>queriers</a>, <a href=#store-gateway>store-gateways</a> and rulers to discover new blocks in the storage.</li></ul><p>For more information, see the <a href=/docs/blocks-storage/compactor/>compactor documentation</a>.</p><p>The compactor is <strong>stateless</strong>.</p><h3 id=store-gateway>Store gateway</h3><p>The <strong>store gateway</strong> is the Cortex service responsible for querying series from blocks. It needs to have an almost up-to-date view of the storage bucket. To discover blocks belonging to their shard, the store gateway can keep the bucket view updated in two different ways:</p><ol><li>Periodically scanning the bucket (default)</li><li>Periodically downloading the <a href=/docs/blocks-storage/bucket-index/>bucket index</a></li></ol><p>For more information, see the <a href=/docs/blocks-storage/store-gateway/>store gateway documentation</a>.</p><p>The store gateway is <strong>semi-stateful</strong>.</p><h3 id=query-frontend>Query frontend</h3><p>The <strong>query frontend</strong> is an <strong>optional service</strong> providing the querier&rsquo;s API endpoints and can be used to accelerate the read path. When the query frontend is in place, incoming query requests should be directed to the query frontend instead of the queriers. The querier service will be still required within the cluster, in order to execute the actual queries.</p><p>The query frontend internally performs some query adjustments and holds queries in an internal queue. In this setup, queriers act as workers which pull jobs from the queue, execute them, and return them to the query-frontend for aggregation. Queriers need to be configured with the query frontend address (via the <code>-querier.frontend-address</code> CLI flag) in order to allow them to connect to the query frontends.</p><p>Query frontends are <strong>stateless</strong>. However, due to how the internal queue works, it&rsquo;s recommended to run a few query frontend replicas to reap the benefit of fair scheduling. Two replicas should suffice in most cases.</p><p>Flow of the query in the system when using query-frontend:</p><ol><li>Query is received by query frontend, which can optionally split it or serve from the cache.</li><li>Query frontend stores the query into in-memory queue, where it waits for some querier to pick it up.</li><li>Querier picks up the query, and executes it.</li><li>Querier sends result back to query-frontend, which then forwards it to the client.</li></ol><p>Query frontend can also be used with any Prometheus-API compatible service. In this mode Cortex can be used as an query accelerator with it&rsquo;s caching and splitting features on other prometheus query engines like Thanos Querier or your own Prometheus server. Query frontend needs to be configured with downstream url address(via the <code>-frontend.downstream-url</code> CLI flag), which is the endpoint of the prometheus server intended to be connected with Cortex.</p><h4 id=queueing>Queueing</h4><p>The query frontend queuing mechanism is used to:</p><ul><li>Ensure that large queries, that could cause an out-of-memory (OOM) error in the querier, will be retried on failure. This allows administrators to under-provision memory for queries, or optimistically run more small queries in parallel, which helps to reduce the total cost of ownership (TCO).</li><li>Prevent multiple large requests from being convoyed on a single querier by distributing them across all queriers using a first-in/first-out queue (FIFO).</li><li>Prevent a single tenant from denial-of-service-ing (DOSing) other tenants by fairly scheduling queries between tenants.</li></ul><h4 id=splitting>Splitting</h4><p>The query frontend splits multi-day queries into multiple single-day queries, executing these queries in parallel on downstream queriers and stitching the results back together again. This prevents large (multi-day) queries from causing out of memory issues in a single querier and helps to execute them faster.</p><h4 id=caching>Caching</h4><p>The query frontend supports caching query results and reuses them on subsequent queries. If the cached results are incomplete, the query frontend calculates the required subqueries and executes them in parallel on downstream queriers. The query frontend can optionally align queries with their step parameter to improve the cacheability of the query results. The result cache is compatible with any cortex caching backend (currently memcached, redis, and an in-memory cache).</p><h3 id=query-scheduler>Query Scheduler</h3><p>Query Scheduler is an <strong>optional</strong> service that moves the internal queue from query frontend into separate component.
diff --git a/docs/blocks-storage/binary-index-header/index.html b/docs/blocks-storage/binary-index-header/index.html
index 0004c4b9ae..e6f5c935e2 100644
--- a/docs/blocks-storage/binary-index-header/index.html
+++ b/docs/blocks-storage/binary-index-header/index.html
@@ -4,7 +4,7 @@
 Store-gateways build the index-header with specific sections of the block&rsquo;s index downloaded using GET byte range requests."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span class=td-sidebar-nav-active-item>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/blocks-storage/binary-index-header.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Binary%20index-header" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/blocks-storage/>Blocks Storage</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/blocks-storage/binary-index-header/ aria-disabled=true class="btn-link disabled">Binary index-header</a></li></ol></nav><div class=td-content><h1>Binary index-header</h1><header class=article-meta></header><p>In order to query series inside blocks from object storage, the <a href=/docs/blocks-storage/store-gateway/>store-gateway</a> has to know certain initial info from each block index. In order to achieve so, the store-gateway builds an index-header for each block and stores it on local disk; such index-header is built by downloading specific pieces of original block&rsquo;s index and storing them on local disk. Index header is then used by store-gateway at query time.</p><p>Store-gateways build the index-header with specific sections of the block&rsquo;s index downloaded using <strong>GET byte range requests</strong>. Since downloading specific sections of the original block&rsquo;s index is a computationally easy operation, the index-header is never uploaded back to the object storage and multiple store-gateway instances (or the same instance after a rolling update without a persistent disk) will re-build the index-header from original block&rsquo;s index each time, if not already existing on local disk.</p><h2 id=format-version-1>Format (version 1)</h2><p>The index-header is a subset of the block index and contains:</p><ul><li><a href=https://github.com/prometheus/prometheus/blob/master/tsdb/docs/format/index.md#symbol-table>Symbol Table</a>, used to unintern string values</li><li><a href=https://github.com/prometheus/prometheus/blob/master/tsdb/docs/format/index.md#postings-offset-table>Posting Offset Table</a>, used to lookup postings</li></ul><p>The following describes the format of the index-header file found in each block store-gateway local directory. It is terminated by a table of contents which serves as an entry point into the index.</p><pre tabindex=0><code>┌─────────────────────────────┬───────────────────────────────┐
diff --git a/docs/blocks-storage/bucket-index/index.html b/docs/blocks-storage/bucket-index/index.html
index 411ed45967..7627315ac1 100644
--- a/docs/blocks-storage/bucket-index/index.html
+++ b/docs/blocks-storage/bucket-index/index.html
@@ -7,7 +7,7 @@
 Benefits The querier, store-gateway and ruler need to have an almost up-to-date view over the entire storage bucket, in order to find the right blocks to lookup at query time (querier) and load block&rsquo;s index-header (store-gateway)."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span class=td-sidebar-nav-active-item>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/blocks-storage/bucket-index.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Bucket%20Index" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#benefits>Benefits</a></li><li><a href=#structure-of-the-index>Structure of the index</a></li><li><a href=#how-it-gets-updated>How it gets updated</a></li><li><a href=#how-its-used-by-the-querier>How it&rsquo;s used by the querier</a></li><li><a href=#how-its-used-by-the-store-gateway>How it&rsquo;s used by the store-gateway</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/blocks-storage/>Blocks Storage</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/blocks-storage/bucket-index/ aria-disabled=true class="btn-link disabled">Bucket Index</a></li></ol></nav><div class=td-content><h1>Bucket Index</h1><header class=article-meta></header><p>The bucket index is a <strong>per-tenant file containing the list of blocks and block deletion marks</strong> in the storage. The bucket index itself is stored in the backend object storage, is periodically updated by the compactor, and used by queriers, store-gateways and rulers to discover blocks in the storage.</p><p>The bucket index usage is <strong>optional</strong> and can be enabled via <code>-blocks-storage.bucket-store.bucket-index.enabled=true</code> (or its respective YAML config option).</p><h2 id=benefits>Benefits</h2><p>The <a href=/docs/blocks-storage/querier/>querier</a>, <a href=/docs/blocks-storage/store-gateway/>store-gateway</a> and ruler need to have an almost up-to-date view over the entire storage bucket, in order to find the right blocks to lookup at query time (querier) and load block&rsquo;s <a href=/docs/blocks-storage/binary-index-header/>index-header</a> (store-gateway). Because of this, they need to periodically scan the bucket to look for new blocks uploaded by ingester or compactor, and blocks deleted (or marked for deletion) by compactor.</p><p>When the bucket index is enabled, the querier, store-gateway and ruler periodically look up the per-tenant bucket index instead of scanning the bucket via &ldquo;list objects&rdquo; operations. This brings few benefits:</p><ol><li>Reduced number of API calls to the object storage by querier and store-gateway</li><li>No &ldquo;list objects&rdquo; storage API calls done by querier and store-gateway</li><li>The <a href=/docs/blocks-storage/querier/>querier</a> is up and running immediately after the startup (no need to run an initial bucket scan)</li></ol><h2 id=structure-of-the-index>Structure of the index</h2><p>The <code>bucket-index.json.gz</code> contains:</p><ul><li><strong><code>blocks</code></strong><br>List of complete blocks of a tenant, including blocks marked for deletion (partial blocks are excluded from the index).</li><li><strong><code>block_deletion_marks</code></strong><br>List of block deletion marks.</li><li><strong><code>updated_at</code></strong><br>Unix timestamp (seconds precision) of when the index has been updated (written in the storage) the last time.</li></ul><h2 id=how-it-gets-updated>How it gets updated</h2><p>The <a href=/docs/blocks-storage/compactor/>compactor</a> periodically scans the bucket and uploads an updated bucket index to the storage. The frequency at which the bucket index is updated can be configured via <code>-compactor.cleanup-interval</code>.</p><p>Despite using the bucket index is optional, the index itself is built and updated by the compactor even if <code>-blocks-storage.bucket-store.bucket-index.enabled</code> has <strong>not</strong> been enabled. This is intentional, so that once a Cortex cluster operator decides to enable the bucket index in a live cluster, the bucket index for any tenant is already existing and query results consistency is guaranteed. The overhead introduced by keeping the bucket index updated is expected to be non significative.</p><h2 id=how-its-used-by-the-querier>How it&rsquo;s used by the querier</h2><p>At query time the <a href=/docs/blocks-storage/querier/>querier</a> and ruler check whether the bucket index for the tenant has already been loaded in memory. If not, the querier and ruler download it from the storage and cache it in memory.</p><p><em>Given it&rsquo;s a small file, lazy downloading it doesn&rsquo;t significantly impact on first query performances, but allows to get a querier up and running without pre-downloading every tenant&rsquo;s bucket index. Moreover, if the <a href=/docs/blocks-storage/querier/#metadata-cache>metadata cache</a> is enabled, the bucket index will be cached for a short time in a shared cache, reducing the actual latency and number of API calls to the object storage in case multiple queriers and rulers will fetch the same tenant&rsquo;s bucket index in a short time.</em></p><p><img src=/images/blocks-storage/bucket-index-querier-workflow.png alt="Querier - Bucket index"></p><p>While in-memory, a background process will keep it <strong>updated at periodic intervals</strong>, so that subsequent queries from the same tenant to the same querier instance will use the cached (and periodically updated) bucket index. There are two config options involved:</p><ul><li><code>-blocks-storage.bucket-store.sync-interval</code><br>This option configures how frequently a cached bucket index should be refreshed.</li><li><code>-blocks-storage.bucket-store.bucket-index.update-on-error-interval</code><br>If downloading a bucket index fails, the failure is cached for a short time in order to avoid hammering the backend storage. This option configures how frequently a bucket index, which previously failed to load, should be tried to load again.</li></ul><p>If a bucket index is unused for a long time (configurable via <code>-blocks-storage.bucket-store.bucket-index.idle-timeout</code>), e.g. because that querier instance is not receiving any query from the tenant, the querier will offload it, stopping to keep it updated at regular intervals. This is particularly for tenants which are resharded to different queriers when <a href=/docs/guides/shuffle-sharding/>shuffle sharding</a> is enabled.</p><p>Finally, at query time the querier and ruler check how old a bucket index is (based on its <code>updated_at</code>) and fail a query if its age is older than <code>-blocks-storage.bucket-store.bucket-index.max-stale-period</code>. This circuit breaker is used to ensure queriers and rulers will not return any partial query results due to a stale view over the long-term storage.</p><h2 id=how-its-used-by-the-store-gateway>How it&rsquo;s used by the store-gateway</h2><p>The <a href=/docs/blocks-storage/store-gateway/>store-gateway</a>, at startup and periodically, fetches the bucket index for each tenant belonging to their shard and uses it as the source of truth for the blocks (and deletion marks) in the storage. This removes the need to periodically scan the bucket to discover blocks belonging to their shard.</p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/blocks-storage/compactor/index.html b/docs/blocks-storage/compactor/index.html
index 195402884f..d0ede743bc 100644
--- a/docs/blocks-storage/compactor/index.html
+++ b/docs/blocks-storage/compactor/index.html
@@ -10,7 +10,7 @@
 How compaction works The blocks compaction has two main benefits:"><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span class=td-sidebar-nav-active-item>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/blocks-storage/compactor.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Compactor" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#how-compaction-works>How compaction works</a></li><li><a href=#compactor-sharding>Compactor sharding</a><ul><li><a href=#waiting-for-stable-ring-at-startup>Waiting for stable ring at startup</a></li></ul></li><li><a href=#soft-and-hard-blocks-deletion>Soft and hard blocks deletion</a></li><li><a href=#compactor-disk-utilization>Compactor disk utilization</a></li><li><a href=#compactor-http-endpoints>Compactor HTTP endpoints</a></li><li><a href=#compactor-configuration>Compactor configuration</a><ul><li><a href=#compactor_config><code>compactor_config</code></a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/blocks-storage/>Blocks Storage</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/blocks-storage/compactor/ aria-disabled=true class="btn-link disabled">Compactor</a></li></ol></nav><div class=td-content><h1>Compactor</h1><header class=article-meta></header><p>The <strong>compactor</strong> is an service which is responsible to:</p><ul><li>Compact multiple blocks of a given tenant into a single optimized larger block. This helps to reduce storage costs (deduplication, index size reduction), and increase query speed (querying fewer blocks is faster).</li><li>Keep the per-tenant bucket index updated. The <a href=/docs/blocks-storage/bucket-index/>bucket index</a> is used by <a href=/docs/blocks-storage/querier/>queriers</a>, <a href=/docs/blocks-storage/store-gateway/>store-gateways</a> and rulers to discover new blocks in the storage.</li></ul><p>The compactor is <strong>stateless</strong>.</p><h2 id=how-compaction-works>How compaction works</h2><p>The blocks compaction has two main benefits:</p><ol><li>Vertically compact blocks uploaded by all ingesters for the same time range</li><li>Horizontally compact blocks with small time ranges into a single larger block</li></ol><p>The <strong>vertical compaction</strong> merges all the blocks of a tenant uploaded by ingesters for the same time range (2 hours ranges by default) into a single block, while <strong>de-duplicating samples</strong> that are originally written to N blocks as a result of replication. This step reduces number of blocks for single 2 hours time range from #(number of ingesters) to 1 per tenant.</p><p>The <strong>horizontal compaction</strong> triggers after the vertical compaction and compacts several blocks with adjacent 2-hour range periods into a single larger block. Even though the total size of block chunks doesn&rsquo;t change after this compaction, it may still significantly reduce the size of the index and the index-header kept in memory by store-gateways.</p><p><img src=/images/blocks-storage/compactor-horizontal-and-vertical-compaction.png alt="Compactor - horizontal and vertical compaction"></p><h2 id=compactor-sharding>Compactor sharding</h2><p>The compactor optionally supports sharding.</p><p>When sharding is enabled, multiple compactor instances can coordinate to split the workload and shard blocks by tenant. All the blocks of a tenant are processed by a single compactor instance at any given time, but compaction for different tenants may simultaneously run on different compactor instances.</p><p>Whenever the pool of compactors increase or decrease (ie. following up a scale up/down), tenants are resharded across the available compactor instances without any manual intervention.</p><p>The compactor sharding is based on the Cortex <a href=/docs/architecture/#the-hash-ring>hash ring</a>. At startup, a compactor generates random tokens and registers itself to the ring. While running, it periodically scans the storage bucket (every <code>-compactor.compaction-interval</code>) to discover the list of tenants in the storage and compacts blocks for each tenant whose hash matches the token ranges assigned to the instance itself within the ring.</p><p>This feature can be enabled via <code>-compactor.sharding-enabled=true</code> and requires the backend <a href=/docs/architecture/#the-hash-ring>hash ring</a> to be configured via <code>-compactor.ring.*</code> flags (or their respective YAML config options).</p><h3 id=waiting-for-stable-ring-at-startup>Waiting for stable ring at startup</h3><p>In the event of a cluster cold start or scale up of 2+ compactor instances at the same time we may end up in a situation where each new compactor instance starts at a slightly different time and thus each one runs the first compaction based on a different state of the ring. This is not a critical condition, but may be inefficient, because multiple compactor replicas may start compacting the same tenant nearly at the same time.</p><p>To reduce the likelihood this could happen, the compactor waits for a stable ring at startup. A ring is considered stable if no instance is added/removed to the ring for at least <code>-compactor.ring.wait-stability-min-duration</code>. If the ring keep getting changed after <code>-compactor.ring.wait-stability-max-duration</code>, the compactor will stop waiting for a stable ring and will proceed starting up normally.</p><p>To disable this waiting logic, you can start the compactor with <code>-compactor.ring.wait-stability-min-duration=0</code>.</p><h2 id=soft-and-hard-blocks-deletion>Soft and hard blocks deletion</h2><p>When the compactor successfully compacts some source blocks into a larger block, source blocks are deleted from the storage. Blocks deletion is not immediate, but follows a two steps process:</p><ol><li>First, a block is <strong>marked for deletion</strong> (soft delete)</li><li>Then, once a block is marked for deletion for longer then <code>-compactor.deletion-delay</code>, the block is <strong>deleted</strong> from the storage (hard delete)</li></ol><p>The compactor is both responsible to mark blocks for deletion and then hard delete them once the deletion delay expires.
diff --git a/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/index.html b/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/index.html
index e039340cf6..efeadd6a83 100644
--- a/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/index.html
+++ b/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/index.html
@@ -5,7 +5,7 @@
 Original design document for blocksconvert is also available."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span class=td-sidebar-nav-active-item>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/blocks-storage/convert-stored-chunks-to-blocks.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Convert%20long-term%20storage%20from%20chunks%20to%20blocks" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#tools>Tools</a><ul><li><a href=#scanner>Scanner</a></li><li><a href=#scheduler>Scheduler</a></li><li><a href=#builder>Builder</a></li><li><a href=#cleaner>Cleaner</a></li><li><a href=#limitations>Limitations</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/blocks-storage/>Blocks Storage</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ aria-disabled=true class="btn-link disabled">Convert long-term storage from chunks to blocks</a></li></ol></nav><div class=td-content><h1>Convert long-term storage from chunks to blocks</h1><header class=article-meta></header><p>If you have <a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/>configured your cluster to write new data to blocks</a>, there is still a question about old data.
diff --git a/docs/blocks-storage/index.html b/docs/blocks-storage/index.html
index df13786510..7621169358 100644
--- a/docs/blocks-storage/index.html
+++ b/docs/blocks-storage/index.html
@@ -1,7 +1,7 @@
 <!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.101.0"><link rel=alternate type=application/rss+xml href=/docs/blocks-storage/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/favicons/android-96x96.png sizes=96x96><link rel=icon type=image/png href=/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/favicons/android-192x192.png sizes=192x192><title>Blocks Storage | Cortex</title><meta name=description content="The blocks storage is a Cortex storage engine based on Prometheus TSDB: it stores each tenant&rsquo;s time series into their own TSDB which write out …"><meta property="og:title" content="Blocks Storage"><meta property="og:description" content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta property="og:type" content="website"><meta property="og:url" content="/docs/blocks-storage/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Blocks Storage"><meta itemprop=description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Blocks Storage"><meta name=twitter:description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span class=td-sidebar-nav-active-item>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/blocks-storage/_index.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Blocks%20Storage" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#architecture>Architecture</a><ul><li><a href=#the-write-path>The write path</a></li><li><a href=#the-read-path>The read path</a></li></ul></li><li><a href=#known-issues>Known issues</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/blocks-storage/ aria-disabled=true class="btn-link disabled">Blocks Storage</a></li></ol></nav><div class=td-content><h1>Blocks Storage</h1><header class=article-meta></header><p>The blocks storage is a Cortex storage engine based on <a href=https://prometheus.io/docs/prometheus/latest/storage/>Prometheus TSDB</a>: it stores each tenant&rsquo;s time series into their own TSDB which write out their series to a on-disk block (defaults to 2h block range periods). Each block is composed by chunk files - containing the timestamp-value pairs for multiple series - and an index, which indexes metric names and labels to time series in the chunk files.</p><p>The supported backends for the blocks storage are:</p><ul><li><a href=https://aws.amazon.com/s3>Amazon S3</a></li><li><a href=https://cloud.google.com/storage/>Google Cloud Storage</a></li><li><a href=https://azure.microsoft.com/en-us/services/storage/>Microsoft Azure Storage</a></li><li><a href=https://wiki.openstack.org/wiki/Swift>OpenStack Swift</a></li><li><a href=https://thanos.io/tip/thanos/storage.md/#filesystem>Local Filesystem</a> (single node only)</li></ul><p><em>Internally, some components are based on <a href=https://thanos.io>Thanos</a>, but no Thanos knowledge is required in order to run it.</em></p><h2 id=architecture>Architecture</h2><p>When running the Cortex blocks storage, the Cortex architecture doesn&rsquo;t significantly change and thus the <a href=/docs/architecture/>general architecture documentation</a> applies to the blocks storage as well. However, there are two additional Cortex services when running the blocks storage:</p><ul><li><a href=/docs/blocks-storage/store-gateway/>Store-gateway</a></li><li><a href=/docs/blocks-storage/compactor/>Compactor</a></li></ul><p><img src=/images/blocks-storage/architecture.png alt=Architecture></p><p>The <strong><a href=/docs/blocks-storage/store-gateway/>store-gateway</a></strong> is responsible to query blocks and is used by the <a href=/docs/blocks-storage/querier/>querier</a> at query time. The store-gateway is required when running the blocks storage.</p><p>The <strong><a href=/docs/blocks-storage/compactor/>compactor</a></strong> is responsible to merge and deduplicate smaller blocks into larger ones, in order to reduce the number of blocks stored in the long-term storage for a given tenant and query them more efficiently. It also keeps the <a href=/docs/blocks-storage/bucket-index/>bucket index</a> updated and, for this reason, it&rsquo;s a required component.</p><p>The <code>alertmanager</code> and <code>ruler</code> components can also use object storage to store its configurations and rules uploaded by users. In that case a separate bucket should be created to store alertmanager configurations and rules: using the same bucket between ruler/alertmanager and blocks will cause issue with the <strong><a href=/docs/blocks-storage/compactor/>compactor</a></strong>.</p><h3 id=the-write-path>The write path</h3><p><strong>Ingesters</strong> receive incoming samples from the distributors. Each push request belongs to a tenant, and the ingester appends the received samples to the specific per-tenant TSDB stored on the local disk. The received samples are both kept in-memory and written to a write-ahead log (WAL) and used to recover the in-memory series in case the ingester abruptly terminates. The per-tenant TSDB is lazily created in each ingester as soon as the first samples are received for that tenant.</p><p>The in-memory samples are periodically flushed to disk - and the WAL truncated - when a new TSDB block is created, which by default occurs every 2 hours. Each newly created block is then uploaded to the long-term storage and kept in the ingester until the configured <code>-blocks-storage.tsdb.retention-period</code> expires, in order to give <a href=/docs/blocks-storage/querier/>queriers</a> and <a href=/docs/blocks-storage/store-gateway/>store-gateways</a> enough time to discover the new block on the storage and download its index-header.</p><p>In order to effectively use the <strong>WAL</strong> and being able to recover the in-memory series upon ingester abruptly termination, the WAL needs to be stored to a persistent disk which can survive in the event of an ingester failure (ie. AWS EBS volume or GCP persistent disk when running in the cloud). For example, if you&rsquo;re running the Cortex cluster in Kubernetes, you may use a StatefulSet with a persistent volume claim for the ingesters. The location on the filesystem where the WAL is stored is the same where local TSDB blocks (compacted from head) are stored and cannot be decoupled. See also the <a href=production-tips/#how-to-estimate--querierquery-store-after>timeline of block uploads</a> and <a href=production-tips/#ingester-disk-space>disk space estimate</a>.</p><h4 id=distributor-series-sharding-and-replication>Distributor series sharding and replication</h4><p>The series sharding and replication done by the distributor doesn&rsquo;t change based on the storage engine.</p><p>It&rsquo;s important to note that due to the replication factor N (typically 3), each time series is stored by N ingesters. Since each ingester writes its own block to the long-term storage, this leads a storage utilization N times more. <a href=/docs/blocks-storage/compactor/>Compactor</a> solves this problem by merging blocks from multiple ingesters into a single block, and removing duplicated samples. After blocks compaction, the storage utilization is significantly smaller.</p><p>For more information, please refer to the following dedicated sections:</p><ul><li><a href=/docs/blocks-storage/compactor/>Compactor</a></li><li><a href=/docs/blocks-storage/production-tips/>Production tips</a></li></ul><h3 id=the-read-path>The read path</h3><p><a href=/docs/blocks-storage/querier/>Queriers</a> and <a href=/docs/blocks-storage/store-gateway/>store-gateways</a> periodically iterate over the storage bucket to discover blocks recently uploaded by ingesters.</p><p>For each discovered block, queriers only download the block&rsquo;s <code>meta.json</code> file (containing some metadata including min and max timestamp of samples within the block), while store-gateways download the <code>meta.json</code> as well as the index-header, which is a small subset of the block&rsquo;s index used by the store-gateway to lookup series at query time.</p><p>Queriers use the blocks metadata to compute the list of blocks that need to be queried at query time and fetch matching series from the store-gateway instances holding the required blocks.</p><p>For more information, please refer to the following dedicated sections:</p><ul><li><a href=/docs/blocks-storage/querier/>Querier</a></li><li><a href=/docs/blocks-storage/store-gateway/>Store-gateway</a></li><li><a href=/docs/blocks-storage/production-tips/>Production tips</a></li></ul><h2 id=known-issues>Known issues</h2><p>GitHub issues tagged with the <a href="https://github.com/cortexproject/cortex/issues?q=is%3Aopen+is%3Aissue+label%3Astorage%2Fblocks"><code>storage/blocks</code></a> label are the best source of currently known issues affecting the blocks storage.</p><h2 id=more-in-this-section>More in this section</h2><ul><li><a href=/docs/blocks-storage/querier/>Querier</a></li><li><a href=/docs/blocks-storage/store-gateway/>Store-gateway</a></li><li><a href=/docs/blocks-storage/compactor/>Compactor</a></li><li><a href=/docs/blocks-storage/production-tips/>Production tips</a></li><li><a href=/docs/blocks-storage/binary-index-header/>Binary index-header</a></li><li><a href=/docs/blocks-storage/bucket-index/>Bucket Index</a></li><li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/>Migrate Cortex cluster from chunks to blocks</a></li><li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/>Convert long-term storage from chunks to blocks</a></li><li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/>Migrate the storage from Thanos and Prometheus</a></li><li><a href=/docs/blocks-storage/learn-more/>Learn more</a></li></ul></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/blocks-storage/index.xml b/docs/blocks-storage/index.xml
index 913aa4baec..674f96d94c 100644
--- a/docs/blocks-storage/index.xml
+++ b/docs/blocks-storage/index.xml
@@ -172,17 +172,17 @@
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">healthcheck_config&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The number of consecutive failed health checks required before&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># considering a target unhealthy. 0 means disabled.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.unhealthy-threshold&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.healthcheck.unhealthy-threshold&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[unhealthy_threshold&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;int&amp;gt; | default = 0]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The approximate amount of time between health checks of an individual&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># target.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.interval&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.healthcheck.interval&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[interval&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;duration&amp;gt; | default = 5s]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The amount of time during which no response from a target means a failed&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># health check.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.timeout&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.healthcheck.timeout&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[timeout&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;duration&amp;gt; | default = 1s]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># If enabled, store gateway query stats will be logged using `info` log level.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
@@ -1373,6 +1373,14 @@
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[lazy_expanded_postings_enabled&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;boolean&amp;gt; | default = false]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Mark posting group as lazy if it fetches more keys than R * max series the&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># query should fetch. With R set to 100, a posting group which fetches 100K&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># keys will be marked as lazy if the current query only fetches 1000 series.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># This config is only valid if lazy expanded posting is enabled. 0 disables&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># the limit.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.lazy-expanded-posting-group-max-key-series-ratio&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[lazy_expanded_posting_group_max_key_series_ratio&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;float&amp;gt; | default = 100]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Controls how many series to fetch per batch in Store Gateway. Default&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># value is 10000.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.series-batch-size&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
@@ -2983,6 +2991,14 @@ Displays the status of the store-gateways ring, including the tokens owned by ea
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[lazy_expanded_postings_enabled&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;boolean&amp;gt; | default = false]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Mark posting group as lazy if it fetches more keys than R * max series the&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># query should fetch. With R set to 100, a posting group which fetches 100K&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># keys will be marked as lazy if the current query only fetches 1000 series.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># This config is only valid if lazy expanded posting is enabled. 0 disables&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># the limit.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.lazy-expanded-posting-group-max-key-series-ratio&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[lazy_expanded_posting_group_max_key_series_ratio&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;float&amp;gt; | default = 100]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Controls how many series to fetch per batch in Store Gateway. Default&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># value is 10000.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.series-batch-size&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
diff --git a/docs/blocks-storage/learn-more/index.html b/docs/blocks-storage/learn-more/index.html
index f4e7f0cd49..7bb5349108 100644
--- a/docs/blocks-storage/learn-more/index.html
+++ b/docs/blocks-storage/learn-more/index.html
@@ -6,7 +6,7 @@
 Talks Scaling Prometheus: How We Got Some Thanos Into Cortex Articles How to switch Cortex from chunks to blocks storage (and why you won’t look back) (Oct 2020) Now GA: Cortex blocks storage for running Prometheus at scale with reduced operational complexity (Oct 2020) Scaling Prometheus: How we’re pushing Cortex blocks storage to its limit and beyond (Aug 2020) How blocks storage in Cortex reduces operational complexity for running Prometheus at massive scale (Jul 2020) "><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span class=td-sidebar-nav-active-item>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/blocks-storage/learn-more.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Learn%20more" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#talks>Talks</a><ul><li><a href=#scaling-prometheus-how-we-got-some-thanos-into-cortex>Scaling Prometheus: How We Got Some Thanos Into Cortex</a></li></ul></li><li><a href=#articles>Articles</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/blocks-storage/>Blocks Storage</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/blocks-storage/learn-more/ aria-disabled=true class="btn-link disabled">Learn more</a></li></ol></nav><div class=td-content><h1>Learn more</h1><header class=article-meta></header><p>In this page we&rsquo;re listing some external resources to learn more about the Cortex blocks storage.</p><h2 id=talks>Talks</h2><h3 id=scaling-prometheus-how-we-got-some-thanos-into-cortex>Scaling Prometheus: How We Got Some Thanos Into Cortex</h3><iframe width=560 height=315 src=https://www.youtube.com/embed/Z5OJzRogAS4 frameborder=0 allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe><h2 id=articles>Articles</h2><ul><li><a href=https://grafana.com/blog/2020/10/19/how-to-switch-cortex-from-chunks-to-blocks-storage-and-why-you-wont-look-back/>How to switch Cortex from chunks to blocks storage (and why you won’t look back)</a> (Oct 2020)</li><li><a href=https://grafana.com/blog/2020/10/06/now-ga-cortex-blocks-storage-for-running-prometheus-at-scale-with-reduced-operational-complexity/>Now GA: Cortex blocks storage for running Prometheus at scale with reduced operational complexity</a> (Oct 2020)</li><li><a href=https://grafana.com/blog/2020/08/12/scaling-prometheus-how-were-pushing-cortex-blocks-storage-to-its-limit-and-beyond/>Scaling Prometheus: How we’re pushing Cortex blocks storage to its limit and beyond</a> (Aug 2020)</li><li><a href=https://grafana.com/blog/2020/07/29/how-blocks-storage-in-cortex-reduces-operational-complexity-for-running-prometheus-at-massive-scale/>How blocks storage in Cortex reduces operational complexity for running Prometheus at massive scale</a> (Jul 2020)</li></ul></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/index.html b/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/index.html
index 12951329fa..0cbfddacf6 100644
--- a/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/index.html
+++ b/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/index.html
@@ -11,7 +11,7 @@
 Cortex cluster is managed by Kubernetes Cortex is using chunks storage Ingesters are using WAL Cortex version 1.4.0 or later. If your ingesters are not using WAL, the documented procedure will still apply, but the presented migration script will not work properly without changes, as it assumes that ingesters are managed via StatefulSet."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span class=td-sidebar-nav-active-item>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/blocks-storage/migrate-from-chunks-to-blocks.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Migrate%20Cortex%20cluster%20from%20chunks%20to%20blocks" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#introduction>Introduction</a></li><li><a href=#step-1-preparation>Step 1: Preparation</a><ul><li><a href=#querier-and-ruler>Querier and Ruler</a></li><li><a href=#compactor-and-store-gateway>Compactor and Store-gateway</a></li><li><a href=#ingester--blocks>Ingester – blocks</a></li><li><a href=#table-manager---chunks>Table-Manager - chunks</a></li></ul></li><li><a href=#step-2-ingesters-migration>Step 2: Ingesters migration</a><ul><li><a href=#known-issues>Known issues</a></li></ul></li><li><a href=#step-3-cleanup>Step 3: Cleanup</a><ul><li></li></ul></li><li><a href=#rollback>Rollback</a><ul><li><a href=#known-issues-1>Known issues</a></li></ul></li><li><a href=#appendix>Appendix</a><ul><li><a href=#jsonnet-config>Jsonnet config</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/blocks-storage/>Blocks Storage</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ aria-disabled=true class="btn-link disabled">Migrate Cortex cluster from chunks to blocks</a></li></ol></nav><div class=td-content><h1>Migrate Cortex cluster from chunks to blocks</h1><header class=article-meta></header><p>This article describes how to migrate existing Cortex cluster from chunks storage to blocks storage,
diff --git a/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/index.html b/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/index.html
index a6c6005efa..281e51ca70 100644
--- a/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/index.html
+++ b/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/index.html
@@ -10,7 +10,7 @@
 Cortex isolates blocks on a per-tenant basis in the bucket and, for this reason, each tenant blocks should be uploaded to a different location in the bucket."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span class=td-sidebar-nav-active-item>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Migrate%20the%20storage%20from%20Thanos%20and%20Prometheus" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#cortex-blocks-storage-requirements>Cortex blocks storage requirements</a></li><li><a href=#how-to-migrate-the-storage>How to migrate the storage</a><ul><li><a href=#upload-tsdb-blocks-to-cortex-bucket>Upload TSDB blocks to Cortex bucket</a></li><li><a href=#migrate-block-metadata-metajson-to-cortex>Migrate block metadata (<code>meta.json</code>) to Cortex</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/blocks-storage/>Blocks Storage</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ aria-disabled=true class="btn-link disabled">Migrate the storage from Thanos and Prometheus</a></li></ol></nav><div class=td-content><h1>Migrate the storage from Thanos and Prometheus</h1><header class=article-meta></header><p>The Cortex blocks storage engine stores series in TSDB blocks uploaded in the storage bucket. This makes very easy to migrate the storage from Thanos and/or Prometheus to Cortex, when running the blocks storage.</p><h2 id=cortex-blocks-storage-requirements>Cortex blocks storage requirements</h2><p>The Cortex blocks storage has few requirements that should be considered when migrating TSDB blocks from Thanos / Prometheus to Cortex:</p><ol><li><strong>The blocks in the bucket should be located at <code>bucket://&lt;tenant-id>/</code></strong><br>Cortex isolates blocks on a per-tenant basis in the bucket and, for this reason, each tenant blocks should be uploaded to a different location in the bucket. The bucket prefix, where a specific tenant blocks should be uploaded, is <code>/&lt;tenant-id>/</code>; if Cortex is running with auth disabled (no multi-tenancy) then the <code>&lt;tenant-id></code> to use is <code>fake</code>.</li><li><strong>Remove Thanos external labels and inject <code>__org_id__</code> into each block&rsquo;s <code>meta.json</code></strong><br>Every block has a little metadata file named <code>meta.json</code>. Thanos stores external labels at <code>thanos</code> > <code>labels</code>, which should be all removed when migrating to Cortex, while the <code>"__org_id__": "&lt;tenant-id>"</code> added.</li></ol><h2 id=how-to-migrate-the-storage>How to migrate the storage</h2><h3 id=upload-tsdb-blocks-to-cortex-bucket>Upload TSDB blocks to Cortex bucket</h3><p>TSDB blocks stored in Prometheus local disk or Thanos bucket should be copied/uploaded to the Cortex bucket at the location <code>bucket://&lt;tenant-id>/</code> (when Cortex is running with auth disabled then <code>&lt;tenant-id></code> must be <code>fake</code>).</p><h3 id=migrate-block-metadata-metajson-to-cortex>Migrate block metadata (<code>meta.json</code>) to Cortex</h3><p>For each block copied/uploaded to the Cortex bucket, there are a few changes required to the <code>meta.json</code>.</p><h4 id=automatically-migrate-metadata-using-the-thanosconvert-tool>Automatically migrate metadata using the <code>thanosconvert</code> tool</h4><p><code>thanosconvert</code> can iterate over a Cortex bucket and make sure that each <code>meta.json</code> has the correct <code>thanos</code> > <code>labels</code> layout.</p><p>⚠ Warning ⚠ <code>thanosconvert</code> will modify files in the bucket you specify. It&rsquo;s recommended that you have backups or enable object versioning before running this tool.</p><p>To run <code>thanosconvert</code>, you need to provide it with the bucket configuration in the same format as the <a href=/docs/configuration/configuration-file/#blocks_storage_config>blocks storage bucket configuration</a>.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-yaml data-lang=yaml><span style=display:flex><span><span style=color:#8f5902;font-style:italic># bucket-config.yaml</span><span style=color:#f8f8f8;text-decoration:underline>
diff --git a/docs/blocks-storage/production-tips/index.html b/docs/blocks-storage/production-tips/index.html
index fc53da2588..260f51dc82 100644
--- a/docs/blocks-storage/production-tips/index.html
+++ b/docs/blocks-storage/production-tips/index.html
@@ -5,7 +5,7 @@
 Ingester Ensure a high number of max open file descriptors The ingester stores received series into per-tenant TSDB blocks. Both TSDB WAL, head and compacted blocks are composed by a relatively large number of files which gets loaded via mmap. This means that the ingester keeps file descriptors open for TSDB WAL segments, chunk files and compacted blocks which haven&rsquo;t reached the retention period yet."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span class=td-sidebar-nav-active-item>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/blocks-storage/production-tips.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Production%20tips" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#ingester>Ingester</a><ul><li><a href=#ensure-a-high-number-of-max-open-file-descriptors>Ensure a high number of max open file descriptors</a></li><li><a href=#ingester-disk-space>Ingester disk space</a></li></ul></li><li><a href=#querier>Querier</a><ul><li><a href=#ensure-caching-is-enabled>Ensure caching is enabled</a></li><li><a href=#ensure-bucket-index-is-enabled>Ensure bucket index is enabled</a></li><li><a href=#avoid-querying-non-compacted-blocks>Avoid querying non compacted blocks</a></li></ul></li><li><a href=#store-gateway>Store-gateway</a><ul><li><a href=#ensure-caching-is-enabled-1>Ensure caching is enabled</a></li><li><a href=#ensure-bucket-index-is-enabled-1>Ensure bucket index is enabled</a></li><li><a href=#ensure-a-high-number-of-max-open-file-descriptors-1>Ensure a high number of max open file descriptors</a></li></ul></li><li><a href=#compactor>Compactor</a><ul><li><a href=#ensure-the-compactor-has-enough-disk-space>Ensure the compactor has enough disk space</a></li></ul></li><li><a href=#caching>Caching</a><ul><li><a href=#ensure-memcached-is-properly-scaled>Ensure memcached is properly scaled</a></li></ul></li><li><a href=#alertmanager>Alertmanager</a><ul><li><a href=#ensure-alertmanager-networking-is-hardened>Ensure Alertmanager networking is hardened</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/blocks-storage/>Blocks Storage</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/blocks-storage/production-tips/ aria-disabled=true class="btn-link disabled">Production tips</a></li></ol></nav><div class=td-content><h1>Production tips</h1><header class=article-meta></header><p>This page shares some tips and things to take in consideration when setting up a production Cortex cluster based on the blocks storage.</p><h2 id=ingester>Ingester</h2><h3 id=ensure-a-high-number-of-max-open-file-descriptors>Ensure a high number of max open file descriptors</h3><p>The ingester stores received series into per-tenant TSDB blocks. Both TSDB WAL, head and compacted blocks are composed by a relatively large number of files which gets loaded via mmap. This means that the ingester keeps file descriptors open for TSDB WAL segments, chunk files and compacted blocks which haven&rsquo;t reached the retention period yet.</p><p>If your Cortex cluster has many tenants or ingester is running with a long <code>-blocks-storage.tsdb.retention-period</code>, the ingester may hit the <strong><code>file-max</code> ulimit</strong> (maximum number of open file descriptions by a process); in such case, we recommend increasing the limit on your system or enabling <a href=/docs/guides/shuffle-sharding/>shuffle sharding</a>.</p><p>The rule of thumb is that a production system shouldn&rsquo;t have the <code>file-max</code> ulimit below <code>65536</code>, but higher values are recommended (eg. <code>1048576</code>).</p><h3 id=ingester-disk-space>Ingester disk space</h3><p>Ingesters create blocks on disk as samples come in, then every 2 hours (configurable) they cut off those blocks and start a new block.</p><p>We typically configure ingesters to retain these blocks for longer, to allow time to recover if something goes wrong uploading to the long-term store and to reduce work in queriers - more detail <a href=#how-to-estimate--querierquery-store-after>here</a>.</p><p>If you configure ingesters with <code>-blocks-storage.tsdb.retention-period=24h</code>, a rule of thumb for disk space required is to take the number of timeseries after replication and multiply by 30KB.</p><p>For example, if you have 20M active series replicated 3 ways, this gives approx 1.7TB. Divide by the number of ingesters and allow some margin for growth, e.g. if you have 20 ingesters then 100GB each should work, or 150GB each to be more comfortable.</p><h2 id=querier>Querier</h2><h3 id=ensure-caching-is-enabled>Ensure caching is enabled</h3><p>The querier relies on caching to reduce the number API calls to the storage bucket. Ensure <a href=/docs/blocks-storage/querier/#caching>caching</a> is properly configured and <a href=#ensure-memcached-is-properly-scaled>properly scaled</a>.</p><h3 id=ensure-bucket-index-is-enabled>Ensure bucket index is enabled</h3><p>The bucket index reduces the number of API calls to the storage bucket and, when enabled, the querier is up and running immediately after the startup (no need to run an initial bucket scan). Ensure <a href=/docs/blocks-storage/bucket-index/>bucket index</a> is enabled for the querier.</p><h3 id=avoid-querying-non-compacted-blocks>Avoid querying non compacted blocks</h3><p>When running Cortex blocks storage cluster at scale, querying non compacted blocks may be inefficient for two reasons:</p><ol><li>Non compacted blocks contain duplicated samples (as effect of the ingested samples replication)</li><li>Overhead introduced querying many small indexes</li></ol><p>Because of this, we would suggest to avoid querying non compacted blocks. In order to do it, you should:</p><ol><li>Run the <a href=/docs/blocks-storage/compactor/>compactor</a></li><li>Configure queriers <code>-querier.query-store-after</code> large enough to give compactor enough time to compact newly uploaded blocks (<em>see below</em>)</li><li>Configure queriers <code>-querier.query-ingesters-within</code> equal to <code>-querier.query-store-after</code> plus 5m (5 minutes is just a delta to query the boundary both from ingesters and queriers)</li><li>Configure ingesters <code>-blocks-storage.tsdb.retention-period</code> at least as <code>-querier.query-ingesters-within</code></li><li>Lower <code>-blocks-storage.bucket-store.ignore-deletion-marks-delay</code> to 1h, otherwise non compacted blocks could be queried anyway, even if their compacted replacement is available</li></ol><h4 id=how-to-estimate--querierquery-store-after>How to estimate <code>-querier.query-store-after</code></h4><p>The <code>-querier.query-store-after</code> should be set to a duration large enough to give compactor enough time to compact newly uploaded blocks, and queriers and store-gateways to discover and sync newly compacted blocks.</p><p>The following diagram shows all the timings involved in the estimation. This diagram should be used only as a template and you&rsquo;re expected to tweak the assumptions based on real measurements in your Cortex cluster. In this example, the following assumptions have been done:</p><ul><li>An ingester takes up to 30 minutes to upload a block to the storage</li><li>The compactor takes up to 3 hours to compact 2h blocks shipped from all ingesters</li><li>Querier and store-gateways take up to 15 minutes to discover and load a new compacted block</li></ul><p>Given these assumptions, in the worst case scenario it would take up to 6h and 45m since when a sample has been ingested until that sample has been appended to a block flushed to the storage and that block has been <a href=/docs/blocks-storage/compactor/>vertically compacted</a> with all other overlapping 2h blocks shipped from ingesters.</p><p><img src=/images/blocks-storage/avoid-querying-non-compacted-blocks.png alt="Avoid querying non compacted blocks"></p><h2 id=store-gateway>Store-gateway</h2><h3 id=ensure-caching-is-enabled-1>Ensure caching is enabled</h3><p>The store-gateway heavily relies on caching both to speed up the queries and to reduce the number of API calls to the storage bucket. Ensure <a href=/docs/blocks-storage/store-gateway/#caching>caching</a> is properly configured and <a href=#ensure-memcached-is-properly-scaled>properly scaled</a>.</p><h3 id=ensure-bucket-index-is-enabled-1>Ensure bucket index is enabled</h3><p>The bucket index reduces the number of API calls to the storage bucket and the startup time of the store-gateway. Ensure <a href=/docs/blocks-storage/bucket-index/>bucket index</a> is enabled for the store-gateway.</p><h3 id=ensure-a-high-number-of-max-open-file-descriptors-1>Ensure a high number of max open file descriptors</h3><p>The store-gateway stores each block’s index-header on the local disk and loads it via mmap. This means that the store-gateway keeps a file descriptor open for each loaded block. If your Cortex cluster has many blocks in the bucket, the store-gateway may hit the <strong><code>file-max</code> ulimit</strong> (maximum number of open file descriptions by a process); in such case, we recommend increasing the limit on your system or running more store-gateway instances with blocks sharding enabled.</p><p>The rule of thumb is that a production system shouldn&rsquo;t have the <code>file-max</code> ulimit below <code>65536</code>, but higher values are recommended (eg. <code>1048576</code>).</p><h2 id=compactor>Compactor</h2><h3 id=ensure-the-compactor-has-enough-disk-space>Ensure the compactor has enough disk space</h3><p>The compactor generally needs a lot of disk space in order to download source blocks from the bucket and store the compacted block before uploading it to the storage. Please refer to <a href=/docs/blocks-storage/compactor/#compactor-disk-utilization>Compactor disk utilization</a> for more information about how to do capacity planning.</p><h2 id=caching>Caching</h2><h3 id=ensure-memcached-is-properly-scaled>Ensure memcached is properly scaled</h3><p>The rule of thumb to ensure memcached is properly scaled is to make sure evictions happen infrequently. When that&rsquo;s not the case and they affect query performances, the suggestion is to scale out the memcached cluster adding more nodes or increasing the memory limit of existing ones.</p><p>We also recommend to run a different memcached cluster for each cache type (metadata, index, chunks). It&rsquo;s not required, but suggested to not worry about the effect of memory pressure on a cache type against others.</p><h2 id=alertmanager>Alertmanager</h2><h3 id=ensure-alertmanager-networking-is-hardened>Ensure Alertmanager networking is hardened</h3><p>If the Alertmanager API is enabled, users with access to Cortex can autonomously configure the Alertmanager, including receiver integrations that allow to issue network requests to the configured URL (eg. webhook). If the Alertmanager network is not hardened, Cortex users may have the ability to issue network requests to any network endpoint including services running in the local network accessible by the Alertmanager itself.</p><p>Despite hardening the system is out of the scope of Cortex, Cortex provides a basic built-in firewall to block connections created by Alertmanager receiver integrations:</p><ul><li><code>-alertmanager.receivers-firewall-block-cidr-networks</code></li><li><code>-alertmanager.receivers-firewall-block-private-addresses</code></li></ul><p><em>These settings can also be overridden on a per-tenant basis via overrides specified in the <a href=/docs/configuration/arguments/#runtime-configuration-file>runtime config</a>.</em></p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/blocks-storage/querier/index.html b/docs/blocks-storage/querier/index.html
index 651ac3b433..4d39feaf5c 100644
--- a/docs/blocks-storage/querier/index.html
+++ b/docs/blocks-storage/querier/index.html
@@ -3,12 +3,12 @@
 The querier is stateless.
 How it works The querier needs to have an almost up-to-date view over the entire storage bucket, in order to find the right blocks to lookup at query time. The querier can keep the bucket view updated in to two different ways:"><meta property="og:type" content="article"><meta property="og:url" content="/docs/blocks-storage/querier/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Querier"><meta itemprop=description content="The querier service handles queries using the PromQL query language. This document dives into the storage-specific details of the querier service. The general architecture documentation applies too.
 The querier is stateless.
-How it works The querier needs to have an almost up-to-date view over the entire storage bucket, in order to find the right blocks to lookup at query time. The querier can keep the bucket view updated in to two different ways:"><meta itemprop=wordCount content="9177"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Querier"><meta name=twitter:description content="The querier service handles queries using the PromQL query language. This document dives into the storage-specific details of the querier service. The general architecture documentation applies too.
+How it works The querier needs to have an almost up-to-date view over the entire storage bucket, in order to find the right blocks to lookup at query time. The querier can keep the bucket view updated in to two different ways:"><meta itemprop=wordCount content="9251"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Querier"><meta name=twitter:description content="The querier service handles queries using the PromQL query language. This document dives into the storage-specific details of the querier service. The general architecture documentation applies too.
 The querier is stateless.
 How it works The querier needs to have an almost up-to-date view over the entire storage bucket, in order to find the right blocks to lookup at query time. The querier can keep the bucket view updated in to two different ways:"><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span class=td-sidebar-nav-active-item>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/blocks-storage/querier.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Querier" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#how-it-works>How it works</a><ul><li><a href=#bucket-index-disabled-default>Bucket index disabled (default)</a></li><li><a href=#bucket-index-enabled>Bucket index enabled</a></li><li><a href=#anatomy-of-a-query-request>Anatomy of a query request</a></li><li><a href=#how-queriers-connect-to-store-gateway>How queriers connect to store-gateway</a></li></ul></li><li><a href=#caching>Caching</a><ul><li><a href=#metadata-cache>Metadata cache</a></li></ul></li><li><a href=#querier-configuration>Querier configuration</a><ul><li><a href=#querier_config><code>querier_config</code></a></li><li><a href=#blocks_storage_config><code>blocks_storage_config</code></a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/blocks-storage/>Blocks Storage</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/blocks-storage/querier/ aria-disabled=true class="btn-link disabled">Querier</a></li></ol></nav><div class=td-content><h1>Querier</h1><header class=article-meta></header><p>The <strong>querier</strong> service handles queries using the <a href=https://prometheus.io/docs/prometheus/latest/querying/basics/>PromQL</a> query language. This document dives into the storage-specific details of the querier service. The <a href=/docs/architecture/#querier>general architecture documentation</a> applies too.</p><p>The querier is <strong>stateless</strong>.</p><h2 id=how-it-works>How it works</h2><p>The querier needs to have an almost up-to-date view over the entire storage bucket, in order to find the right blocks to lookup at query time. The querier can keep the bucket view updated in to two different ways:</p><ol><li>Periodically scanning the bucket (default)</li><li>Periodically downloading the <a href=/docs/blocks-storage/bucket-index/>bucket index</a></li></ol><h3 id=bucket-index-disabled-default>Bucket index disabled (default)</h3><p>At startup, <strong>queriers</strong> iterate over the entire storage bucket to discover all tenants blocks and download the <code>meta.json</code> for each block. During this initial bucket scanning phase, a querier is not ready to handle incoming queries yet and its <code>/ready</code> readiness probe endpoint will fail.</p><p>While running, queriers periodically iterate over the storage bucket to discover new tenants and recently uploaded blocks. Queriers do <strong>not</strong> download any content from blocks except a small <code>meta.json</code> file containing the block&rsquo;s metadata (including the minimum and maximum timestamp of samples within the block).</p><p>Queriers use the metadata to compute the list of blocks that need to be queried at query time and fetch matching series from the <a href=/docs/blocks-storage/store-gateway/>store-gateway</a> instances holding the required blocks.</p><h3 id=bucket-index-enabled>Bucket index enabled</h3><p>When <a href=/docs/blocks-storage/bucket-index/>bucket index</a> is enabled, queriers lazily download the bucket index upon the first query received for a given tenant, cache it in memory and periodically keep it update. The bucket index contains the list of blocks and block deletion marks of a tenant, which is later used during the query execution to find the set of blocks that need to be queried for the given query.</p><p>Given the bucket index removes the need to scan the bucket, it brings few benefits:</p><ol><li>The querier is expected to be ready shortly after startup.</li><li>Lower volume of API calls to object storage.</li></ol><h3 id=anatomy-of-a-query-request>Anatomy of a query request</h3><p>When a querier receives a query range request, it contains the following parameters:</p><ul><li><code>query</code>: the PromQL query expression itself (e.g. <code>rate(node_cpu_seconds_total[1m])</code>)</li><li><code>start</code>: the start time</li><li><code>end</code>: the end time</li><li><code>step</code>: the query resolution (e.g. <code>30</code> to have 1 resulting data point every 30s)</li></ul><p>Given a query, the querier analyzes the <code>start</code> and <code>end</code> time range to compute a list of all known blocks containing at least 1 sample within this time range. Given the list of blocks, the querier then computes a list of store-gateway instances holding these blocks and sends a request to each matching store-gateway instance asking to fetch all the samples for the series matching the <code>query</code> within the <code>start</code> and <code>end</code> time range.</p><p>The request sent to each store-gateway contains the list of block IDs that are expected to be queried, and the response sent back by the store-gateway to the querier contains the list of block IDs that were actually queried. This list may be a subset of the requested blocks, for example due to recent blocks resharding event (ie. last few seconds). The querier runs a consistency check on responses received from the store-gateways to ensure all expected blocks have been queried; if not, the querier retries to fetch samples from missing blocks from different store-gateways (if the <code>-store-gateway.sharding-ring.replication-factor</code> is greater than <code>1</code>) and if the consistency check fails after all retries, the query execution fails as well (correctness is always guaranteed).</p><p>If the query time range covers a period within <code>-querier.query-ingesters-within</code> duration, the querier also sends the request to all ingesters, in order to fetch samples that have not been uploaded to the long-term storage yet.</p><p>Once all samples have been fetched from both store-gateways and ingesters, the querier proceeds with running the PromQL engine to execute the query and send back the result to the client.</p><h3 id=how-queriers-connect-to-store-gateway>How queriers connect to store-gateway</h3><p>Queriers need to discover store-gateways in order to connect to them at query time. The service discovery mechanism used depends whether blocks sharding is enabled in the store-gateways.</p><p>When blocks sharding is <strong>enabled</strong>, queriers need to access to the store-gateways hash ring and thus queriers need to be configured with the same <code>-store-gateway.sharding-ring.*</code> flags (or their respective YAML config options) store-gateways have been configured.</p><p>When blocks sharding is <strong>disabled</strong>, queriers need the <code>-querier.store-gateway-addresses</code> CLI flag (or its respective YAML config option) being set to a comma separated list of store-gateway addresses in <a href=/docs/configuration/arguments/#dns-service-discovery>DNS Service Discovery format</a>. Queriers will evenly balance the requests to query blocks across the resolved addresses.</p><h2 id=caching>Caching</h2><p>The querier supports the following caches:</p><ul><li><a href=#metadata-cache>Metadata cache</a></li></ul><p>Caching is optional, but <strong>highly recommended</strong> in a production environment. Please also check out the <a href=/docs/blocks-storage/production-tips/#caching>production tips</a> for more information about configuring the cache.</p><h3 id=metadata-cache>Metadata cache</h3><p><a href=/docs/blocks-storage/store-gateway/>Store-gateway</a> and querier can use memcached for caching bucket metadata:</p><ul><li>List of tenants</li><li>List of blocks per tenant</li><li>Block&rsquo;s <code>meta.json</code> content</li><li>Block&rsquo;s <code>deletion-mark.json</code> existence and content</li><li>Tenant&rsquo;s <code>bucket-index.json.gz</code> content</li></ul><p>Using the metadata cache can significantly reduce the number of API calls to object storage and protects from linearly scale the number of these API calls with the number of querier and store-gateway instances (because the bucket is periodically scanned and synched by each querier and store-gateway).</p><p>To enable metadata cache, please set <code>-blocks-storage.bucket-store.metadata-cache.backend</code>. Only <code>memcached</code> backend is supported currently. Memcached client has additional configuration available via flags with <code>-blocks-storage.bucket-store.metadata-cache.memcached.*</code> prefix.</p><p>Additional options for configuring metadata cache have <code>-blocks-storage.bucket-store.metadata-cache.*</code> prefix. By configuring TTL to zero or negative value, caching of given item type is disabled.</p><p><em>The same memcached backend cluster should be shared between store-gateways and queriers.</em></p><h2 id=querier-configuration>Querier configuration</h2><p>This section described the querier configuration. For the general Cortex configuration and references to common config blocks, please refer to the <a href=/docs/configuration/configuration-file/>configuration documentation</a>.</p><h3 id=querier_config><code>querier_config</code></h3><p>The <code>querier_config</code> configures the Cortex querier.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-yaml data-lang=yaml><span style=display:flex><span><span style=color:#204a87;font-weight:700>querier</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline>
@@ -125,17 +125,17 @@
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#204a87;font-weight:700>healthcheck_config</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#8f5902;font-style:italic># The number of consecutive failed health checks required before</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#8f5902;font-style:italic># considering a target unhealthy. 0 means disabled.</span><span style=color:#f8f8f8;text-decoration:underline>
-</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#8f5902;font-style:italic># CLI flag: -querier.store-gateway-client.unhealthy-threshold</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#8f5902;font-style:italic># CLI flag: -querier.store-gateway-client.healthcheck.unhealthy-threshold</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#204a87;font-weight:700>[unhealthy_threshold</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;int&gt; | default = 0]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#8f5902;font-style:italic># The approximate amount of time between health checks of an individual</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#8f5902;font-style:italic># target.</span><span style=color:#f8f8f8;text-decoration:underline>
-</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#8f5902;font-style:italic># CLI flag: -querier.store-gateway-client.interval</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#8f5902;font-style:italic># CLI flag: -querier.store-gateway-client.healthcheck.interval</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#204a87;font-weight:700>[interval</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;duration&gt; | default = 5s]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#8f5902;font-style:italic># The amount of time during which no response from a target means a failed</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#8f5902;font-style:italic># health check.</span><span style=color:#f8f8f8;text-decoration:underline>
-</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#8f5902;font-style:italic># CLI flag: -querier.store-gateway-client.timeout</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#8f5902;font-style:italic># CLI flag: -querier.store-gateway-client.healthcheck.timeout</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>      </span><span style=color:#204a87;font-weight:700>[timeout</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;duration&gt; | default = 1s]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># If enabled, store gateway query stats will be logged using `info` log level.</span><span style=color:#f8f8f8;text-decoration:underline>
@@ -1324,6 +1324,14 @@
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#204a87;font-weight:700>[lazy_expanded_postings_enabled</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;boolean&gt; | default = false]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># Mark posting group as lazy if it fetches more keys than R * max series the</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># query should fetch. With R set to 100, a posting group which fetches 100K</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># keys will be marked as lazy if the current query only fetches 1000 series.</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># This config is only valid if lazy expanded posting is enabled. 0 disables</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># the limit.</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -blocks-storage.bucket-store.lazy-expanded-posting-group-max-key-series-ratio</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#204a87;font-weight:700>[lazy_expanded_posting_group_max_key_series_ratio</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;float&gt; | default = 100]</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># Controls how many series to fetch per batch in Store Gateway. Default</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># value is 10000.</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -blocks-storage.bucket-store.series-batch-size</span><span style=color:#f8f8f8;text-decoration:underline>
diff --git a/docs/blocks-storage/store-gateway/index.html b/docs/blocks-storage/store-gateway/index.html
index a20cfc5966..27f047107f 100644
--- a/docs/blocks-storage/store-gateway/index.html
+++ b/docs/blocks-storage/store-gateway/index.html
@@ -5,13 +5,13 @@
 Periodically scanning the bucket (default) Periodically downloading the bucket index Bucket index disabled (default) At startup store-gateways iterate over the entire storage bucket to discover blocks for all tenants and download the meta."><meta property="og:type" content="article"><meta property="og:url" content="/docs/blocks-storage/store-gateway/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Store-gateway"><meta itemprop=description content="The store-gateway is the Cortex service responsible to query series from blocks. The store-gateway is required when running the Cortex blocks storage.
 The store-gateway is semi-stateful.
 How it works The store-gateway needs to have an almost up-to-date view over the storage bucket, in order to discover blocks belonging to their shard. The store-gateway can keep the bucket view updated in to two different ways:
-Periodically scanning the bucket (default) Periodically downloading the bucket index Bucket index disabled (default) At startup store-gateways iterate over the entire storage bucket to discover blocks for all tenants and download the meta."><meta itemprop=wordCount content="10192"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Store-gateway"><meta name=twitter:description content="The store-gateway is the Cortex service responsible to query series from blocks. The store-gateway is required when running the Cortex blocks storage.
+Periodically scanning the bucket (default) Periodically downloading the bucket index Bucket index disabled (default) At startup store-gateways iterate over the entire storage bucket to discover blocks for all tenants and download the meta."><meta itemprop=wordCount content="10266"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Store-gateway"><meta name=twitter:description content="The store-gateway is the Cortex service responsible to query series from blocks. The store-gateway is required when running the Cortex blocks storage.
 The store-gateway is semi-stateful.
 How it works The store-gateway needs to have an almost up-to-date view over the storage bucket, in order to discover blocks belonging to their shard. The store-gateway can keep the bucket view updated in to two different ways:
 Periodically scanning the bucket (default) Periodically downloading the bucket index Bucket index disabled (default) At startup store-gateways iterate over the entire storage bucket to discover blocks for all tenants and download the meta."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span class=td-sidebar-nav-active-item>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/blocks-storage/store-gateway.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Store-gateway" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#how-it-works>How it works</a><ul><li><a href=#bucket-index-disabled-default>Bucket index disabled (default)</a></li><li><a href=#bucket-index-enabled>Bucket index enabled</a></li></ul></li><li><a href=#blocks-sharding-and-replication>Blocks sharding and replication</a><ul><li><a href=#sharding-strategies>Sharding strategies</a></li><li><a href=#auto-forget>Auto-forget</a></li><li><a href=#zone-awareness>Zone-awareness</a></li><li><a href=#waiting-for-stable-ring-at-startup>Waiting for stable ring at startup</a></li></ul></li><li><a href=#blocks-index-header>Blocks index-header</a><ul><li><a href=#index-header-lazy-loading>Index-header lazy loading</a></li></ul></li><li><a href=#caching>Caching</a><ul><li><a href=#index-cache>Index cache</a></li><li><a href=#chunks-cache>Chunks cache</a></li><li><a href=#metadata-cache>Metadata cache</a></li></ul></li><li><a href=#store-gateway-http-endpoints>Store-gateway HTTP endpoints</a></li><li><a href=#store-gateway-configuration>Store-gateway configuration</a><ul><li><a href=#store_gateway_config><code>store_gateway_config</code></a></li><li><a href=#blocks_storage_config><code>blocks_storage_config</code></a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/blocks-storage/>Blocks Storage</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/blocks-storage/store-gateway/ aria-disabled=true class="btn-link disabled">Store-gateway</a></li></ol></nav><div class=td-content><h1>Store-gateway</h1><header class=article-meta></header><p>The store-gateway is the Cortex service responsible to query series from blocks. The store-gateway is required when running the Cortex blocks storage.</p><p>The store-gateway is <strong>semi-stateful</strong>.</p><h2 id=how-it-works>How it works</h2><p>The store-gateway needs to have an almost up-to-date view over the storage bucket, in order to discover blocks belonging to their shard. The store-gateway can keep the bucket view updated in to two different ways:</p><ol><li>Periodically scanning the bucket (default)</li><li>Periodically downloading the <a href=/docs/blocks-storage/bucket-index/>bucket index</a></li></ol><h3 id=bucket-index-disabled-default>Bucket index disabled (default)</h3><p>At startup <strong>store-gateways</strong> iterate over the entire storage bucket to discover blocks for all tenants and download the <code>meta.json</code> and index-header for each block. During this initial bucket synchronization phase, the store-gateway <code>/ready</code> readiness probe endpoint will fail.</p><p>While running, store-gateways periodically rescan the storage bucket to discover new blocks (uploaded by the ingesters and <a href=/docs/blocks-storage/compactor/>compactor</a>) and blocks marked for deletion or fully deleted since the last scan (as a result of compaction). The frequency at which this occurs is configured via <code>-blocks-storage.bucket-store.sync-interval</code>.</p><p>The blocks chunks and the entire index are never fully downloaded by the store-gateway. The index-header is stored to the local disk, in order to avoid to re-download it on subsequent restarts of a store-gateway. For this reason, it&rsquo;s recommended - but not required - to run the store-gateway with a persistent disk. For example, if you&rsquo;re running the Cortex cluster in Kubernetes, you may use a StatefulSet with a persistent volume claim for the store-gateways.</p><p><em>For more information about the index-header, please refer to <a href=/docs/blocks-storage/binary-index-header/>Binary index-header documentation</a>.</em></p><h3 id=bucket-index-enabled>Bucket index enabled</h3><p>When bucket index is enabled, the overall workflow is the same but, instead of iterating over the bucket objects, the store-gateway fetch the <a href=/docs/blocks-storage/bucket-index/>bucket index</a> for each tenant belonging to their shard in order to discover each tenant&rsquo;s blocks and block deletion marks.</p><p><em>For more information about the bucket index, please refer to <a href=/docs/blocks-storage/bucket-index/>bucket index documentation</a>.</em></p><h2 id=blocks-sharding-and-replication>Blocks sharding and replication</h2><p>The store-gateway optionally supports blocks sharding. Sharding can be used to horizontally scale blocks in a large cluster without hitting any vertical scalability limit.</p><p>When sharding is enabled, store-gateway instances builds an <a href=/docs/architecture/#the-hash-ring>hash ring</a> and blocks get sharded and replicated across the pool of store-gateway instances registered within the ring.</p><p>Store-gateways continuously monitor the ring state and whenever the ring topology changes (e.g. a new instance has been added/removed or gets healthy/unhealthy) each store-gateway instance resync the blocks assigned to its shard, based on the block ID hash matching the token ranges assigned to the instance itself within the ring.</p><p>For each block belonging to a store-gateway shard, the store-gateway loads its <code>meta.json</code>, the <code>deletion-mark.json</code> and the index-header. Once a block is loaded on the store-gateway, it&rsquo;s ready to be queried by queriers. When the querier queries blocks through a store-gateway, the response will contain the list of actually queried block IDs. If a querier tries to query a block which has not been loaded by a store-gateway, the querier will either retry on a different store-gateway (if blocks replication is enabled) or fail the query.</p><p>Blocks can be replicated across multiple store-gateway instances based on a replication factor configured via <code>-store-gateway.sharding-ring.replication-factor</code>. The blocks replication is used to protect from query failures caused by some blocks not loaded by any store-gateway instance at a given time like, for example, in the event of a store-gateway failure or while restarting a store-gateway instance (e.g. during a rolling update).</p><p>This feature can be enabled via <code>-store-gateway.sharding-enabled=true</code> and requires the backend <a href=/docs/architecture/#the-hash-ring>hash ring</a> to be configured via <code>-store-gateway.sharding-ring.*</code> flags (or their respective YAML config options).</p><h3 id=sharding-strategies>Sharding strategies</h3><p>The store-gateway supports two sharding strategies:</p><ul><li><code>default</code></li><li><code>shuffle-sharding</code></li><li><code>zone-stable-shuffle-sharding</code></li></ul><p>The <strong><code>default</code></strong> sharding strategy spreads the blocks of each tenant across all store-gateway instances. It&rsquo;s the easiest form of sharding supported, but doesn&rsquo;t provide any workload isolation between different tenants.</p><p>The <strong><code>shuffle-sharding</code></strong> strategy spreads the blocks of a tenant across a subset of store-gateway instances. This way, the number of store-gateway instances loading blocks of a single tenant is limited and the blast radius of any issue that could be introduced by the tenant&rsquo;s workload is limited to its shard instances.</p><p>The shuffle sharding strategy can be enabled via <code>-store-gateway.sharding-strategy=shuffle-sharding</code> and requires the <code>-store-gateway.tenant-shard-size</code> flag (or their respective YAML config options) to be set to the default shard size, which is the default number of store-gateway instances each tenant should be sharded to. The shard size can then be overridden on a per-tenant basis setting the <code>store_gateway_tenant_shard_size</code> in the limits overrides.</p><p>The <strong><code>zone-stable-shuffle-sharding</code></strong> strategy achieves the same as the <strong><code>shuffle-sharding</code></strong> strategy, but using a different sharding algorithm. The new sharding algorithm ensures that when zone awareness is enabled, when shard size increases or decreases by one, the replicas for any block should only change at most by one instance. This is important for querying store gateway because a block can be retried at most 3 times.</p><p>Zone stable shuffle sharding can be enabled via <code>-store-gateway.sharding-ring.zone-stable-shuffle-sharding</code> CLI flag.</p><p>It will become the default shuffle sharding strategy for store gateway in <code>v1.17.0</code> release and the previous shuffle sharding algorithm will be removed in <code>v1.18.0</code> release.</p><p><em>Please check out the <a href=/docs/guides/shuffle-sharding/>shuffle sharding documentation</a> for more information about how it works.</em></p><h3 id=auto-forget>Auto-forget</h3><p>When a store-gateway instance cleanly shutdowns, it automatically unregisters itself from the ring. However, in the event of a crash or node failure, the instance will not be unregistered from the ring, potentially leaving a spurious entry in the ring forever.</p><p>To protect from this, when an healthy store-gateway instance finds another instance in the ring which is unhealthy for more than 10 times the configured <code>-store-gateway.sharding-ring.heartbeat-timeout</code>, the healthy instance forcibly removes the unhealthy one from the ring.</p><p>This feature is called <strong>auto-forget</strong> and is built into the store-gateway.</p><h3 id=zone-awareness>Zone-awareness</h3><p>The store-gateway replication optionally supports <a href=/docs/guides/zone-aware-replication/>zone-awareness</a>. When zone-aware replication is enabled and the blocks replication factor is > 1, each block is guaranteed to be replicated across store-gateway instances running in different availability zones.</p><p><strong>To enable</strong> the zone-aware replication for the store-gateways you should:</p><ol><li>Configure the availability zone for each store-gateway via the <code>-store-gateway.sharding-ring.instance-availability-zone</code> CLI flag (or its respective YAML config option)</li><li>Enable blocks zone-aware replication via the <code>-store-gateway.sharding-ring.zone-awareness-enabled</code> CLI flag (or its respective YAML config option). Please be aware this configuration option should be set to store-gateways, queriers and rulers.</li><li>Rollout store-gateways, queriers and rulers to apply the new configuration</li></ol><h3 id=waiting-for-stable-ring-at-startup>Waiting for stable ring at startup</h3><p>In the event of a cluster cold start or scale up of 2+ store-gateway instances at the same time we may end up in a situation where each new store-gateway instance starts at a slightly different time and thus each one runs the initial blocks sync based on a different state of the ring. For example, in case of a cold start, the first store-gateway joining the ring may load all blocks since the sharding logic runs based on the current state of the ring, which is 1 single store-gateway.</p><p>To reduce the likelihood this could happen, the store-gateway waits for a stable ring at startup. A ring is considered stable if no instance is added/removed to the ring for at least <code>-store-gateway.sharding-ring.wait-stability-min-duration</code>. If the ring keep getting changed after <code>-store-gateway.sharding-ring.wait-stability-max-duration</code>, the store-gateway will stop waiting for a stable ring and will proceed starting up normally.</p><p>To disable this waiting logic, you can start the store-gateway with <code>-store-gateway.sharding-ring.wait-stability-min-duration=0</code>.</p><h2 id=blocks-index-header>Blocks index-header</h2><p>The <a href=/docs/blocks-storage/binary-index-header/>index-header</a> is a subset of the block index which the store-gateway downloads from the object storage and keeps on the local disk in order to speed up queries.</p><p>At startup, the store-gateway downloads the index-header of each block belonging to its shard. A store-gateway is not ready until this initial index-header download is completed. Moreover, while running, the store-gateway periodically looks for newly uploaded blocks in the storage and downloads the index-header for the blocks belonging to its shard.</p><h3 id=index-header-lazy-loading>Index-header lazy loading</h3><p>By default, each index-header is memory mapped by the store-gateway right after downloading it. In a cluster with a large number of blocks, each store-gateway may have a large amount of memory mapped index-headers, regardless how frequently they&rsquo;re used at query time.</p><p>Cortex supports a configuration option <code>-blocks-storage.bucket-store.index-header-lazy-loading-enabled=true</code> to enable index-header lazy loading. When enabled, index-headers will be memory mapped only once required by a query and will be automatically released after <code>-blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout</code> time of inactivity.</p><h2 id=caching>Caching</h2><p>The store-gateway supports the following caches:</p><ul><li><a href=#index-cache>Index cache</a></li><li><a href=#chunks-cache>Chunks cache</a></li><li><a href=#metadata-cache>Metadata cache</a></li></ul><p>Caching is optional, but <strong>highly recommended</strong> in a production environment. Please also check out the <a href=/docs/blocks-storage/production-tips/#caching>production tips</a> for more information about configuring the cache.</p><h3 id=index-cache>Index cache</h3><p>The store-gateway can use a cache to speed up lookups of postings and series from TSDB blocks indexes. Two backends are supported:</p><ul><li><code>inmemory</code></li><li><code>memcached</code></li><li><code>redis</code></li></ul><h4 id=in-memory-index-cache>In-memory index cache</h4><p>The <code>inmemory</code> index cache is <strong>enabled by default</strong> and its max size can be configured through the flag <code>-blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes</code> (or config file). The trade-off of using the in-memory index cache is:</p><ul><li>Pros: zero latency</li><li>Cons: increased store-gateway memory usage, not shared across multiple store-gateway replicas (when sharding is disabled or replication factor > 1)</li></ul><h4 id=memcached-index-cache>Memcached index cache</h4><p>The <code>memcached</code> index cache allows to use <a href=https://memcached.org/>Memcached</a> as cache backend. This cache backend is configured using <code>-blocks-storage.bucket-store.index-cache.backend=memcached</code> and requires the Memcached server(s) addresses via <code>-blocks-storage.bucket-store.index-cache.memcached.addresses</code> (or config file). The addresses are resolved using the <a href=/docs/configuration/arguments/#dns-service-discovery>DNS service provider</a>.</p><p>The trade-off of using the Memcached index cache is:</p><ul><li>Pros: can scale beyond a single node memory (Memcached cluster), shared across multiple store-gateway instances</li><li>Cons: higher latency in the cache round trip compared to the in-memory one</li></ul><p>The Memcached client uses a jump hash algorithm to shard cached entries across a cluster of Memcached servers. For this reason, you should make sure memcached servers are <strong>not</strong> behind any kind of load balancer and their address is configured so that servers are added/removed to the end of the list whenever a scale up/down occurs.</p><p>For example, if you&rsquo;re running Memcached in Kubernetes, you may:</p><ol><li>Deploy your Memcached cluster using a <a href=https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/>StatefulSet</a></li><li>Create a <a href=https://kubernetes.io/docs/concepts/services-networking/service/#headless-services>headless service</a> for Memcached StatefulSet</li><li>Configure the Cortex&rsquo;s Memcached client address using the <code>dnssrvnoa+</code> <a href=/docs/configuration/arguments/#dns-service-discovery>service discovery</a></li></ol><h4 id=redis-index-cache>Redis index cache</h4><p>The <code>redis</code> index cache allows to use <a href=https://memcached.org/>Redis</a> as cache backend. This cache backend is configured using <code>-blocks-storage.bucket-store.index-cache.backend=redis</code> and requires the Redis server(s) addresses via <code>-blocks-storage.bucket-store.index-cache.redis.addresses</code> (or config file).</p><p>Using <code>redis</code> as the cache backend has similar trade-offs as using <code>memcached</code> cache backend. However, client side caching can be enabled when using <code>redis</code> backend to avoid Store Gateway fetching data from cache each time. See <a href=https://redis.io/docs/manual/client-side-caching/>here</a> for more info and it can be enabled by setting flag <code>-blocks-storage.bucket-store.index-cache.redis.cache-size</code> > 0.</p><h3 id=chunks-cache>Chunks cache</h3><p>Store-gateway can also use a cache for storing chunks fetched from the storage. Chunks contain actual samples, and can be reused if user query hits the same series for the same time range.</p><p>To enable chunks cache, please set <code>-blocks-storage.bucket-store.chunks-cache.backend</code>. Chunks can be stored into Memcached or Redis cache. Memcached client can be configured via flags with <code>-blocks-storage.bucket-store.chunks-cache.memcached.*</code> prefix. Redis client can be configured via flags with <code>-blocks-storage.bucket-store.chunks-cache.redis.*</code> prefix.</p><p>There are additional low-level options for configuring chunks cache. Please refer to other flags with <code>-blocks-storage.bucket-store.chunks-cache.*</code> prefix.</p><h3 id=metadata-cache>Metadata cache</h3><p>Store-gateway and <a href=/docs/blocks-storage/querier/>querier</a> can use memcached or redis for caching bucket metadata:</p><ul><li>List of tenants</li><li>List of blocks per tenant</li><li>Block&rsquo;s <code>meta.json</code> content</li><li>Block&rsquo;s <code>deletion-mark.json</code> existence and content</li><li>Tenant&rsquo;s <code>bucket-index.json.gz</code> content</li></ul><p>Using the metadata cache can significantly reduce the number of API calls to object storage and protects from linearly scale the number of these API calls with the number of querier and store-gateway instances (because the bucket is periodically scanned and synched by each querier and store-gateway).</p><p>To enable metadata cache, please set <code>-blocks-storage.bucket-store.metadata-cache.backend</code>. <code>memcached</code> and <code>redis</code> backend are supported currently. Memcached client has additional configuration available via flags with <code>-blocks-storage.bucket-store.metadata-cache.memcached.*</code> prefix. Redis client has additional configuration available via flags with <code>-blocks-storage.bucket-store.metadata-cache.redis.*</code> prefix.</p><p>Additional options for configuring metadata cache have <code>-blocks-storage.bucket-store.metadata-cache.*</code> prefix. By configuring TTL to zero or negative value, caching of given item type is disabled.</p><p><em>The same cache backend deployment should be shared between store-gateways and queriers.</em></p><h2 id=store-gateway-http-endpoints>Store-gateway HTTP endpoints</h2><ul><li><code>GET /store-gateway/ring</code><br>Displays the status of the store-gateways ring, including the tokens owned by each store-gateway and an option to remove (forget) instances from the ring.</li></ul><h2 id=store-gateway-configuration>Store-gateway configuration</h2><p>This section described the store-gateway configuration. For the general Cortex configuration and references to common config blocks, please refer to the <a href=/docs/configuration/configuration-file/>configuration documentation</a>.</p><h3 id=store_gateway_config><code>store_gateway_config</code></h3><p>The <code>store_gateway_config</code> configures the store-gateway service used by the blocks storage.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-yaml data-lang=yaml><span style=display:flex><span><span style=color:#204a87;font-weight:700>store_gateway</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline>
@@ -1330,6 +1330,14 @@
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#204a87;font-weight:700>[lazy_expanded_postings_enabled</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;boolean&gt; | default = false]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># Mark posting group as lazy if it fetches more keys than R * max series the</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># query should fetch. With R set to 100, a posting group which fetches 100K</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># keys will be marked as lazy if the current query only fetches 1000 series.</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># This config is only valid if lazy expanded posting is enabled. 0 disables</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># the limit.</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -blocks-storage.bucket-store.lazy-expanded-posting-group-max-key-series-ratio</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#204a87;font-weight:700>[lazy_expanded_posting_group_max_key_series_ratio</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;float&gt; | default = 100]</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># Controls how many series to fetch per batch in Store Gateway. Default</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># value is 10000.</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -blocks-storage.bucket-store.series-batch-size</span><span style=color:#f8f8f8;text-decoration:underline>
diff --git a/docs/case-studies/buoyant-cloud/index.html b/docs/case-studies/buoyant-cloud/index.html
index ea9fc24942..93eaae8020 100644
--- a/docs/case-studies/buoyant-cloud/index.html
+++ b/docs/case-studies/buoyant-cloud/index.html
@@ -4,7 +4,7 @@
 Linkerd is an open source service mesh for Kubernetes, and part of the CNCF, along with Prometheus and Cortex. Linkerd provides three key classes of features: observability, reliability, and security—all without requiring any changes to your code. That first pillar, observability, has motivated deep Prometheus integration, and is the focus of this case study."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span class=td-sidebar-nav-active-item>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/case-studies/buoyant-cloud.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Buoyant%20Cloud%20and%20Cortex:%20Standing%20on%20the%20shoulders%20of%20Linkerd%20and%20Prometheus" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#linkerd-x-prometheus>Linkerd x Prometheus</a></li><li><a href=#buoyant-x-cortex>Buoyant x Cortex</a><ul><li><a href=#buoyant-cloud-prototyping-with-prometheus>Buoyant Cloud Prototyping with Prometheus</a></li><li><a href=#observability-requirements>Observability Requirements</a></li><li><a href=#buoyant-cloud-production-ready-with-cortex>Buoyant Cloud Production-Ready with Cortex</a></li></ul></li><li><a href=#looking-ahead>Looking Ahead</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/case-studies/>Case Studies</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/case-studies/buoyant-cloud/ aria-disabled=true class="btn-link disabled">Buoyant Cloud</a></li></ol></nav><div class=td-content><h1>Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus</h1><header class=article-meta></header><p><a href=https://buoyant.io>Buoyant</a>, the creator of <a href=https://linkerd.io>Linkerd</a>, has had a close relationship with Prometheus and related technologies for years. As of today, that relationship now includes <a href=https://cortexmetrics.io/>Cortex</a>.</p><p>Linkerd is an open source service mesh for Kubernetes, and part of the CNCF, along with Prometheus and Cortex. Linkerd provides three key classes of features: observability, reliability, and security—all without requiring any changes to your code. That first pillar, observability, has motivated deep Prometheus integration, and is the focus of this case study.</p><h2 id=linkerd-x-prometheus>Linkerd x Prometheus</h2><p>In 2016, Linkerd 1.0 shipped with a <a href=https://linkerd.io/2016/10/04/a-service-mesh-for-kubernetes-part-i-top-line-service-metrics/#step-3-install-linkerd-viz>visualization module</a> built with Prometheus and Grafana. With the release of <a href=https://linkerd.io/2018/09/18/announcing-linkerd-2-0/>Linkerd 2.0 in 2018</a>, Prometheus and Grafana were promoted to first-class components. That evolution was documented in a blog post entitled <a href=https://linkerd.io/2018/05/17/prometheus-the-right-way-lessons-learned-evolving-conduits-prometheus-integration/>Prometheus the Right Way</a>, and a KubeCon North America 2018 talk entitled <a href="https://www.youtube.com/watch?v=bnDWApsH36Y">Linkerd 2.0, Now with Extra Prometheus</a>, both co-authored by <a href=https://twitter.com/fredbrancz>Frederic Branczyk</a> of Polar Signals and <a href=https://twitter.com/siggy>Andrew Seigner</a> of Buoyant.</p><p>This deep integration in Linkerd 2.0 provided out-of-the-box golden metrics, that is, success rate, request rate, and latency, across all your Kubernetes workloads.</p><p><img src=/images/case-studies/bcloud-linkerd-grafana.png alt="Linkerd Grafana">
diff --git a/docs/case-studies/gojek/index.html b/docs/case-studies/gojek/index.html
index ccd4600197..3919d57604 100644
--- a/docs/case-studies/gojek/index.html
+++ b/docs/case-studies/gojek/index.html
@@ -4,7 +4,7 @@
 The company’s mission is to solve everyday challenges with technology innovation."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span class=td-sidebar-nav-active-item>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/case-studies/gojek.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=How%20Gojek%20Is%20Leveraging%20Cortex%20to%20Keep%20Up%20with%20Its%20Ever-Growing%20Scale" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/case-studies/>Case Studies</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/case-studies/gojek/ aria-disabled=true class="btn-link disabled">Gojek</a></li></ol></nav><div class=td-content><h1>How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale</h1><header class=article-meta></header><p><a href=https://www.gojek.com/>Gojek</a> launched in 2010 as a call center for booking motorcycle taxi rides in Indonesia. Today, the startup is a decacorn serving millions of users across Southeast Asia with its mobile wallet, GoPay, and 20+ products on its super app. Want to order dinner? Book a massage? Buy movie tickets? You can do all of that with the Gojek app.</p><p>The company’s mission is to solve everyday challenges with technology innovation. To achieve that across multiple markets the systems team at Gojek focused on building an infrastructure for speed, reliability, and scale. By 2019, the team realized it needed a new monitoring system that could keep up with Gojek’s ever-growing technology organization, which led them to <a href=https://github.com/cortexproject/cortex>Cortex</a>, the horizontally scalable <a href=https://prometheus.io/>Prometheus</a> implementation.</p><p>“We were using InfluxDB for metrics storage. Developers configured alerts by committing kapacitor scripts in git repos. To achieve high availability, we had a relay setup with two InfluxDBs. Since we could not horizontally scale Influx unless we paid for an enterprise license, we ended up having many InfluxDB clusters with relay setup,” says Product Engineer Ankit Goel.</p><p>Though the team had introduced automation for setup, managing all those Influx instances became a pain point for operations. Additionally, some of the Gojek engineering teams needed far greater scale. “Some of our teams generate more than a million active time series,” says Goel. Another common requirement from customers was long-term storage of metrics. With InfluxDB, Gojek only had 2 weeks’ retention, and increasing it would mean provisioning bigger instances.</p><p>Gojek was in search of a better monitoring solution that would meet the following requirements:</p><ul><li>Kubernetes native.</li><li>Horizontally scalable.</li><li>Highly available out of the box.</li><li>High reliability.</li><li>Low operations overhead so a small team can manage it.</li></ul><p>Cortex met all of these requirements, and also had the following features that the Gojek team could leverage:</p><ul><li>Multi-tenancy.</li><li>Customizable and modifiable, so it could be integrated with Gojek’s existing tooling.</li><li>Support for remote_write.</li></ul><p>Because it supports remote_write, Cortex enabled one of Gojek’s key needs: the ability to offer monitoring as a service. “With Thanos, we would have had to deploy a Thanos sidecar on every Prometheus that would have been deployed,” says Goel. “So essentially, there would be a substantial part of infrastructure on the client side that we would need to manage. We preferred Cortex because people could simply push their metrics to us, and we would have all the metrics in a single place.”</p><p>The implementation started in January 2019. The team developed a few tools: a simple service for token-based authentication, and another for storing team information, such as notification channels and PagerDuty policies. Once all this was done, they leveraged <a href=https://github.com/achilles42/telegraf/tree/prometheus-remote-write>InfluxData Telegraf’s remote_write plugin</a> to write to Cortex. This allowed them to have all the metrics being sent to InfluxDB to be sent to Cortex as well. “So moving from InfluxDB to tenants would not be that complicated. Because Cortex was multi-tenant, we could directly map each of our InfluxDB servers to our tenants,” says Goel. They’ve developed an internal helm chart to deploy and manage Cortex. After the customizations were completed in about two months, “we had our setup up and running, and we onboarded one team on Cortex,” he says.</p><p>In the initial version, GitOps was the only interface for developers to apply alerts and create dashboards. The team built tools like <a href=https://github.com/lahsivjar/grafonnet-playground>grafonnet-playground</a> to make it easy for developers to create dashboards. Developers are also allowed to create dashboards using the UI, since Grafana maintains version history for dashboards.</p><p><img src=/images/case-studies/gojek-jsonnet-playground.png alt=grafonnet-playground></p><p>“We needed metrics like ‘the number of alerts triggered for each team,’ ‘how long did it take to resolve these alerts,’ ‘how many were actionable and how many were ignored,’ etc.,” says Goel. “For measuring these metrics, the team only had to create a simple dashboard, since the ruler component exposes the per-tenant alert metrics. Both business and developers have found these metrics to be very useful.”</p><p><img src=/images/case-studies/gojek-alerting-analytics.png alt="alert analytics"></p><p>The team built a CLI tool to improve user experience for applying alerts without having to dig into PromQL. “You can write a command and say <code>lens attach alert</code>, and you tell it what kind of alert you want to attach, such as a CPU alert or Postgres alerts, and then you give it a service name,” says Goel. “There are some challenges to this approach for applying alerts, but we would like to move to such a model in the future.”</p><p>One of the challenges the team faces is developer education. But “we always knew if we are going to move to either Thanos or Cortex, developers would have to learn PromQL,” Goel says. The monitoring team paired with developers to help them understand PromQL and migrate their graphs and alerts.</p><p>The monitoring team has faced issues with Cortex from time to time, but “we always reached out to the Cortex community with our issues through the Cortex slack channel,” says Goel, and “active members of the Cortex community have always helped us with our problems.”</p><p>Today, Gojek’s Lens monitoring system has 40+ tenants, for which Cortex handles about <strong>1.2 million samples per second</strong>. Adoption is growing organically by word of mouth. Gojek is currently migrating to Kubernetes, and the teams that moved to Kubernetes have found Prometheus to be a better fit than InfluxDB. Seeing that success, other teams on Kubernetes have onboarded themselves to Lens.</p><p><img src=/images/case-studies/gojek-throughput.png alt="samples per second"></p><p>Ultimately, Goel says, “where Cortex has really helped us is to integrate the monitoring system with our existing tools. We have a lot of internal tooling, and in certain places, we needed really tight integrations with the monitoring system. So the goal is to make sure that whenever a new service or team is created, they automatically get onboarded to the monitoring platform. After developers deploy, some of their system metrics and all the other standard metrics that are available for a service are automatically sent to the platform.” The team plans to spend the next six months bringing everyone over to Lens.</p><p>Looking ahead, Goel and his team have the long-term vision of growing from a monitoring team to a full-fledged observability team. “We also want to take care of logging and tracing in Gojek,” he says. “<a href=https://github.com/grafana/loki>Loki</a> would be easy to fit with Cortex, so in the future we want to explore Loki for logging.”</p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/case-studies/index.html b/docs/case-studies/index.html
index 51046a7c8c..28555fd7b5 100644
--- a/docs/case-studies/index.html
+++ b/docs/case-studies/index.html
@@ -1,7 +1,7 @@
 <!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.101.0"><link rel=alternate type=application/rss+xml href=/docs/case-studies/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/favicons/android-96x96.png sizes=96x96><link rel=icon type=image/png href=/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/favicons/android-192x192.png sizes=192x192><title>Case Studies | Cortex</title><meta name=description content><meta property="og:title" content="Case Studies"><meta property="og:description" content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta property="og:type" content="website"><meta property="og:url" content="/docs/case-studies/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Case Studies"><meta itemprop=description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Case Studies"><meta name=twitter:description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span class=td-sidebar-nav-active-item>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/case-studies/_index.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Case%20Studies" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/case-studies/ aria-disabled=true class="btn-link disabled">Case Studies</a></li></ol></nav><div class=td-content><h1>Case Studies</h1><header class=article-meta></header><ul><li><a href=/docs/case-studies/gojek/>How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale</a></li><li><a href=/docs/case-studies/rewe-digital/>How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic</a></li><li><a href=/docs/case-studies/buoyant-cloud/>Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus</a></li></ul></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/case-studies/rewe-digital/index.html b/docs/case-studies/rewe-digital/index.html
index 34c98e895b..4ba8739ca1 100644
--- a/docs/case-studies/rewe-digital/index.html
+++ b/docs/case-studies/rewe-digital/index.html
@@ -4,7 +4,7 @@
 The REWE digital subsidiary was started in 2014 to advance its parent company’s digital transformation, so “we have a rather modern tech stack,” says Cloud Platform Engineer Martin Schneppenheim."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span class=td-sidebar-nav-active-item>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/case-studies/rewe-digital.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=How%20Cortex%20helped%20REWE%20digital%20ensure%20stability%20while%20scaling%20services%20during%20the%20Covid-19%20pandemic" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#searching-for-a-scalable-monitoring-solution>Searching for a scalable monitoring solution</a></li><li><a href=#implementation>Implementation</a></li><li><a href=#results>Results</a></li><li><a href=#looking-ahead>Looking ahead</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/case-studies/>Case Studies</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/case-studies/rewe-digital/ aria-disabled=true class="btn-link disabled">REWE digital</a></li></ol></nav><div class=td-content><h1>How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic</h1><header class=article-meta></header><p><a href=https://www.rewe-digital.com/>REWE digital</a> builds the technology that drives the e-commerce, app, and food pickup and delivery services for one of Germany’s largest grocery chains, REWE. Like other companies involved in the food supply chain, REWE has seen demand spike during the Covid-19 pandemic. Thanks to its adoption of Cortex last year, the monitoring team has been able to ensure stability at growing scale.</p><p>The REWE digital subsidiary was started in 2014 to advance its parent company’s digital transformation, so “we have a rather modern tech stack,” says Cloud Platform Engineer Martin Schneppenheim. A lot of the platform is run on Kubernetes using GKE while some is still running on-prem using Nomad. Still, there were some challenges that arose from the Spotify tribe model that the organization adopted. Each of the four tribes at REWE digital &ndash; ECOM, FULFILLMENT, CONTENT and PLATFORM, and while the tribes mostly converged on the same technologies, he says, “each platform team has its own solution for, say, Kafka, for Prometheus, for Grafana.”</p><p>Plus, over the past six years, the company has grown from 30 employees to around 600.</p><p>With this rapid growth, they realized by the end of 2018 that they needed a new solution. The tipping point came when they experienced some out-of-memory issues with Prometheus, he says, “and we had no idea why.”</p><p>Each tribe had one Prometheus HA pair that was used by all the teams in the tribe. One of the tribes used one Prometheus pair which required 30-60 gigabytes of RAM per instance. “We still saw some random out-of-memory kills, and we believe it was because some queries were loading too many samples,” Schneppenheim says. “We had several platform teams doing basically the same thing, and we wanted to tackle such issues organization-wide.”</p><p><img src=/images/case-studies/rewe-workplace.jpg alt="Grafana powered by Cortex"></p><h3 id=searching-for-a-scalable-monitoring-solution>Searching for a scalable monitoring solution</h3><p>The solution needed to support the Prometheus format, since all of REWE’s microservices had a Prometheus end point. And the team wanted to have trust in the project’s longevity. After considering M3, Victoria Metrics, and Thanos, the REWE digital team decided to go with Cortex. “Cortex had just been released as a CNCF project, and there were several developers from different companies,” he says. “That was another plus point for us.”
diff --git a/docs/changelog/index.html b/docs/changelog/index.html
index 9bad1fd02e..9116a03347 100644
--- a/docs/changelog/index.html
+++ b/docs/changelog/index.html
@@ -3,13 +3,13 @@
 [CHANGE] OTLP: Change OTLP handler to be consistent with the Prometheus OTLP handler. #6272
 
 
-target_info metric is enabled by …"><meta property="og:title" content="Changelog"><meta property="og:description" content="master / unreleased [CHANGE] OTLP: Change OTLP handler to be consistent with the Prometheus OTLP handler. #6272 target_info metric is enabled by default and can be disabled via -distributor.otlp.disable-target-info=true flag Convert all attributes to labels is disabled by default and can be enabled via -distributor.otlp.convert-all-attributes=true flag You can specify the attributes converted to labels via -distributor.promote-resource-attributes flag. Supported only if -distributor.otlp.convert-all-attributes=false [CHANGE] Change all max async concurrency default values 50 to 3 #6268 [CHANGE] Change default value of -blocks-storage."><meta property="og:type" content="article"><meta property="og:url" content="/docs/changelog/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Changelog"><meta itemprop=description content="master / unreleased [CHANGE] OTLP: Change OTLP handler to be consistent with the Prometheus OTLP handler. #6272 target_info metric is enabled by default and can be disabled via -distributor.otlp.disable-target-info=true flag Convert all attributes to labels is disabled by default and can be enabled via -distributor.otlp.convert-all-attributes=true flag You can specify the attributes converted to labels via -distributor.promote-resource-attributes flag. Supported only if -distributor.otlp.convert-all-attributes=false [CHANGE] Change all max async concurrency default values 50 to 3 #6268 [CHANGE] Change default value of -blocks-storage."><meta itemprop=wordCount content="27438"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Changelog"><meta name=twitter:description content="master / unreleased [CHANGE] OTLP: Change OTLP handler to be consistent with the Prometheus OTLP handler. #6272 target_info metric is enabled by default and can be disabled via -distributor.otlp.disable-target-info=true flag Convert all attributes to labels is disabled by default and can be enabled via -distributor.otlp.convert-all-attributes=true flag You can specify the attributes converted to labels via -distributor.promote-resource-attributes flag. Supported only if -distributor.otlp.convert-all-attributes=false [CHANGE] Change all max async concurrency default values 50 to 3 #6268 [CHANGE] Change default value of -blocks-storage."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
+target_info metric is enabled by …"><meta property="og:title" content="Changelog"><meta property="og:description" content="master / unreleased [CHANGE] OTLP: Change OTLP handler to be consistent with the Prometheus OTLP handler. #6272 target_info metric is enabled by default and can be disabled via -distributor.otlp.disable-target-info=true flag Convert all attributes to labels is disabled by default and can be enabled via -distributor.otlp.convert-all-attributes=true flag You can specify the attributes converted to labels via -distributor.promote-resource-attributes flag. Supported only if -distributor.otlp.convert-all-attributes=false [CHANGE] Change all max async concurrency default values 50 to 3 #6268 [CHANGE] Change default value of -blocks-storage."><meta property="og:type" content="article"><meta property="og:url" content="/docs/changelog/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Changelog"><meta itemprop=description content="master / unreleased [CHANGE] OTLP: Change OTLP handler to be consistent with the Prometheus OTLP handler. #6272 target_info metric is enabled by default and can be disabled via -distributor.otlp.disable-target-info=true flag Convert all attributes to labels is disabled by default and can be enabled via -distributor.otlp.convert-all-attributes=true flag You can specify the attributes converted to labels via -distributor.promote-resource-attributes flag. Supported only if -distributor.otlp.convert-all-attributes=false [CHANGE] Change all max async concurrency default values 50 to 3 #6268 [CHANGE] Change default value of -blocks-storage."><meta itemprop=wordCount content="27473"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Changelog"><meta name=twitter:description content="master / unreleased [CHANGE] OTLP: Change OTLP handler to be consistent with the Prometheus OTLP handler. #6272 target_info metric is enabled by default and can be disabled via -distributor.otlp.disable-target-info=true flag Convert all attributes to labels is disabled by default and can be enabled via -distributor.otlp.convert-all-attributes=true flag You can specify the attributes converted to labels via -distributor.promote-resource-attributes flag. Supported only if -distributor.otlp.convert-all-attributes=false [CHANGE] Change all max async concurrency default values 50 to 3 #6268 [CHANGE] Change default value of -blocks-storage."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span class=td-sidebar-nav-active-item>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/CHANGELOG.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Changelog" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
-<a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#master--unreleased>master / unreleased</a></li><li><a href=#1181-2024-10-14>1.18.1 2024-10-14</a></li><li><a href=#1180-2024-09-03>1.18.0 2024-09-03</a></li><li><a href=#1171-2024-05-20>1.17.1 2024-05-20</a></li><li><a href=#1170-2024-04-30>1.17.0 2024-04-30</a></li><li><a href=#1161-2024-04-23>1.16.1 2024-04-23</a></li><li><a href=#1160-2023-11-20>1.16.0 2023-11-20</a></li><li><a href=#1153-2023-06-22>1.15.3 2023-06-22</a></li><li><a href=#1152-2023-05-09>1.15.2 2023-05-09</a></li><li><a href=#1151-2023-04-26>1.15.1 2023-04-26</a></li><li><a href=#1150-2023-04-19>1.15.0 2023-04-19</a></li><li><a href=#1140-2022-12-02>1.14.0 2022-12-02</a><ul><li><a href=#known-issues>Known issues</a></li></ul></li><li><a href=#1130-2022-07-14>1.13.0 2022-07-14</a></li><li><a href=#1110-2021-11-25>1.11.0 2021-11-25</a></li><li><a href=#1100--2021-08-03>1.10.0 / 2021-08-03</a></li><li><a href=#blocksconvert>Blocksconvert</a></li><li><a href=#190--2021-05-14>1.9.0 / 2021-05-14</a></li><li><a href=#blocksconvert-1>Blocksconvert</a></li><li><a href=#181--2021-04-27>1.8.1 / 2021-04-27</a></li><li><a href=#180--2021-03-24>1.8.0 / 2021-03-24</a></li><li><a href=#171--2021-04-27>1.7.1 / 2021-04-27</a></li><li><a href=#170--2021-02-23>1.7.0 / 2021-02-23</a></li><li><a href=#160--2020-12-29>1.6.0 / 2020-12-29</a></li><li><a href=#blocksconvert-2>Blocksconvert</a></li><li><a href=#150--2020-11-09>1.5.0 / 2020-11-09</a><ul><li><a href=#cortex>Cortex</a></li><li><a href=#blocksconvert-3>Blocksconvert</a></li></ul></li><li><a href=#140--2020-10-02>1.4.0 / 2020-10-02</a></li><li><a href=#130--2020-08-21>1.3.0 / 2020-08-21</a></li><li><a href=#120--2020-07-01>1.2.0 / 2020-07-01</a></li><li><a href=#110--2020-05-21>1.1.0 / 2020-05-21</a></li><li><a href=#101--2020-04-23>1.0.1 / 2020-04-23</a></li><li><a href=#100--2020-04-02>1.0.0 / 2020-04-02</a><ul><li><a href=#known-issues-1>Known issues</a></li><li><a href=#config-file-breaking-changes>Config file breaking changes</a></li></ul></li><li><a href=#070--2020-03-16>0.7.0 / 2020-03-16</a><ul><li><a href=#full-changelog>Full changelog</a></li><li><a href=#changes-to-denormalised-tokens-in-the-ring>Changes to denormalised tokens in the ring</a></li><li><a href=#known-issues-2>Known issues</a></li><li><a href=#annotated-config-file-breaking-changes>Annotated config file breaking changes</a></li></ul></li><li><a href=#061--2020-02-05>0.6.1 / 2020-02-05</a></li><li><a href=#060--2020-01-28>0.6.0 / 2020-01-28</a><ul><li><a href=#upgrading-postgresql-if-youre-using-configs-service>Upgrading PostgreSQL (if you&rsquo;re using configs service)</a></li><li><a href=#known-issues-3>Known issues</a></li></ul></li><li><a href=#040--2019-12-02>0.4.0 / 2019-12-02</a></li><li><a href=#030--2019-10-11>0.3.0 / 2019-10-11</a></li><li><a href=#020--2019-09-05>0.2.0 / 2019-09-05</a></li><li><a href=#010--2019-08-07>0.1.0 / 2019-08-07</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/changelog/ aria-disabled=true class="btn-link disabled">Changelog</a></li></ol></nav><div class=td-content><h1>Changelog</h1><header class=article-meta></header><h2 id=master--unreleased>master / unreleased</h2><ul><li>[CHANGE] OTLP: Change OTLP handler to be consistent with the Prometheus OTLP handler. <a href=https://github.com/cortexproject/cortex/pull/6272>#6272</a></li></ul><ul><li><code>target_info</code> metric is enabled by default and can be disabled via <code>-distributor.otlp.disable-target-info=true</code> flag</li><li>Convert all attributes to labels is disabled by default and can be enabled via <code>-distributor.otlp.convert-all-attributes=true</code> flag</li><li>You can specify the attributes converted to labels via <code>-distributor.promote-resource-attributes</code> flag. Supported only if <code>-distributor.otlp.convert-all-attributes=false</code></li></ul><ul><li>[CHANGE] Change all max async concurrency default values <code>50</code> to <code>3</code> <a href=https://github.com/cortexproject/cortex/pull/6268>#6268</a></li><li>[CHANGE] Change default value of <code>-blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency</code> from <code>50</code> to <code>3</code> <a href=https://github.com/cortexproject/cortex/pull/6265>#6265</a></li><li>[CHANGE] Enable Compactor and Alertmanager in target all. <a href=https://github.com/cortexproject/cortex/pull/6204>#6204</a></li><li>[FEATURE] Ruler: Pagination support for List Rules API. <a href=https://github.com/cortexproject/cortex/pull/6299>#6299</a></li><li>[FEATURE] Query Frontend/Querier: Add protobuf codec <code>-api.querier-default-codec</code> and the option to choose response compression type <code>-querier.response-compression</code>. <a href=https://github.com/cortexproject/cortex/pull/5527>#5527</a></li><li>[FEATURE] Ruler: Experimental: Add <code>ruler.frontend-address</code> to allow query to query frontends instead of ingesters. <a href=https://github.com/cortexproject/cortex/pull/6151>#6151</a></li><li>[FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via <code>-ruler.enable-ha-evaluation</code> flag. <a href=https://github.com/cortexproject/cortex/pull/6129>#6129</a></li><li>[FEATURE] Store Gateway: Add an in-memory chunk cache. <a href=https://github.com/cortexproject/cortex/pull/6245>#6245</a></li><li>[FEATURE] Chunk Cache: Support multi level cache and add metrics. <a href=https://github.com/cortexproject/cortex/pull/6249>#6249</a></li><li>[FEATURE] Distributor: Accept multiple HA Tracker pairs in the same request. <a href=https://github.com/cortexproject/cortex/pull/6256>#6256</a></li><li>[FEATURE] Ruler: Add support for per-user external labels <a href=https://github.com/cortexproject/cortex/pull/6340>#6340</a></li><li>[ENHANCEMENT] Store Gateway: Add a hedged request to reduce the tail latency. <a href=https://github.com/cortexproject/cortex/pull/6388>#6388</a></li><li>[ENHANCEMENT] Ingester: Add metrics to track succeed/failed native histograms. <a href=https://github.com/cortexproject/cortex/pull/6370>#6370</a></li><li>[ENHANCEMENT] Query Frontend/Querier: Add an experimental flag <code>-querier.enable-promql-experimental-functions</code> to enable experimental promQL functions. <a href=https://github.com/cortexproject/cortex/pull/6355>#6355</a></li><li>[ENHANCEMENT] OTLP: Add <code>-distributor.otlp-max-recv-msg-size</code> flag to limit OTLP request size in bytes. <a href=https://github.com/cortexproject/cortex/pull/6333>#6333</a></li><li>[ENHANCEMENT] S3 Bucket Client: Add a list objects version configs to configure list api object version. <a href=https://github.com/cortexproject/cortex/pull/6280>#6280</a></li><li>[ENHANCEMENT] OpenStack Swift: Add application credential configs for Openstack swift object storage backend. <a href=https://github.com/cortexproject/cortex/pull/6255>#6255</a></li><li>[ENHANCEMENT] Query Frontend: Add new query stats metrics <code>cortex_query_samples_scanned_total</code> and <code>cortex_query_peak_samples</code> to track scannedSamples and peakSample per user. <a href=https://github.com/cortexproject/cortex/pull/6228>#6228</a></li><li>[ENHANCEMENT] Ingester: Add option <code>ingester.disable-chunk-trimming</code> to disable chunk trimming. <a href=https://github.com/cortexproject/cortex/pull/6300>#6300</a></li><li>[ENHANCEMENT] Ingester: Add <code>blocks-storage.tsdb.wal-compression-type</code> to support zstd wal compression type. <a href=https://github.com/cortexproject/cortex/pull/6232>#6232</a></li><li>[ENHANCEMENT] Query Frontend: Add info field to query response. <a href=https://github.com/cortexproject/cortex/pull/6207>#6207</a></li><li>[ENHANCEMENT] Query Frontend: Add peakSample in query stats response. <a href=https://github.com/cortexproject/cortex/pull/6188>#6188</a></li><li>[ENHANCEMENT] Ruler: Add new ruler metric <code>cortex_ruler_rule_groups_in_store</code> that is the total rule groups per tenant in store, which can be used to compare with <code>cortex_prometheus_rule_group_rules</code> to count the number of rule groups that are not loaded by a ruler. <a href=https://github.com/cortexproject/cortex/pull/5869>#5869</a></li><li>[ENHANCEMENT] Ingester/Ring: New <code>READONLY</code> status on ring to be used by Ingester. New ingester API to change mode of ingester <a href=https://github.com/cortexproject/cortex/pull/6163>#6163</a></li><li>[ENHANCEMENT] Ruler: Add query statistics metrics when &ndash;ruler.query-stats-enabled=true. <a href=https://github.com/cortexproject/cortex/pull/6173>#6173</a></li><li>[ENHANCEMENT] Ingester: Add new API <code>/ingester/all_user_stats</code> which shows loaded blocks, active timeseries and ingestion rate for a specific ingester. <a href=https://github.com/cortexproject/cortex/pull/6178>#6178</a></li><li>[ENHANCEMENT] Distributor: Add new <code>cortex_reduced_resolution_histogram_samples_total</code> metric to track the number of histogram samples which resolution was reduced. <a href=https://github.com/cortexproject/cortex/pull/6182>#6182</a></li><li>[ENHANCEMENT] StoreGateway: Implement metadata API limit in queryable. <a href=https://github.com/cortexproject/cortex/pull/6195>#6195</a></li><li>[ENHANCEMENT] Ingester: Add matchers to ingester LabelNames() and LabelNamesStream() RPC. <a href=https://github.com/cortexproject/cortex/pull/6209>#6209</a></li><li>[ENHANCEMENT] KV: Add TLS configs to consul. <a href=https://github.com/cortexproject/cortex/pull/6374>#6374</a></li><li>[ENHANCEMENT] Ingester/Store Gateway Clients: Introduce an experimental HealthCheck handler to quickly fail requests directed to unhealthy targets. <a href=https://github.com/cortexproject/cortex/pull/6225>#6225</a> <a href=https://github.com/cortexproject/cortex/pull/6257>#6257</a></li><li>[ENHANCEMENT] Upgrade build image and Go version to 1.23.2. <a href=https://github.com/cortexproject/cortex/pull/6261>#6261</a> <a href=https://github.com/cortexproject/cortex/pull/6262>#6262</a></li><li>[ENHANCEMENT] Ingester: Introduce a new experimental feature for caching expanded postings on the ingester. <a href=https://github.com/cortexproject/cortex/pull/6296>#6296</a></li><li>[ENHANCEMENT] Querier/Ruler: Expose <code>store_gateway_consistency_check_max_attempts</code> for max retries when querying store gateway in consistency check. <a href=https://github.com/cortexproject/cortex/pull/6276>#6276</a></li><li>[ENHANCEMENT] StoreGateway: Add new <code>cortex_bucket_store_chunk_pool_inuse_bytes</code> metric to track the usage in chunk pool. <a href=https://github.com/cortexproject/cortex/pull/6310>#6310</a></li><li>[ENHANCEMENT] Distributor: Add new <code>cortex_distributor_inflight_client_requests</code> metric to track number of ingester client inflight requests. <a href=https://github.com/cortexproject/cortex/pull/6358>#6358</a></li><li>[ENHANCEMENT] Distributor: Expose <code>cortex_label_size_bytes</code> native histogram metric. <a href=https://github.com/cortexproject/cortex/pull/6372>#6372</a></li><li>[ENHANCEMENT] Add new option <code>-server.grpc_server-num-stream-workers</code> to configure the number of worker goroutines that should be used to process incoming streams. <a href=https://github.com/cortexproject/cortex/pull/6386>#6386</a></li><li>[ENHANCEMENT] Distributor: Return HTTP 5XX instead of HTTP 4XX when instance limits are hit. <a href=https://github.com/cortexproject/cortex/pull/6358>#6358</a></li><li>[ENHANCEMENT] Ingester: Make sure unregistered ingester joining the ring after WAL replay <a href=https://github.com/cortexproject/cortex/pull/6277>#6277</a></li><li>[BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / <a href=https://github.com/cortexproject/cortex/pull/6224>#6224</a></li><li>[BUGFIX] Ruler: Allow rule evaluation to complete during shutdown. <a href=https://github.com/cortexproject/cortex/pull/6326>#6326</a></li><li>[BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled <a href=https://github.com/cortexproject/cortex/pull/6271>#6271</a></li><li>[BUGFIX] ingester: Fix regression on usage of cortex_ingester_queried_chunks <a href=https://github.com/cortexproject/cortex/pull/6398>#6398</a></li></ul><h2 id=1181-2024-10-14>1.18.1 2024-10-14</h2><ul><li>[BUGFIX] Backporting upgrade to go 1.22.7 to patch CVE-2024-34155, CVE-2024-34156, CVE-2024-34158 <a href=https://github.com/cortexproject/cortex/pull/6217>#6217</a> <a href=https://github.com/cortexproject/cortex/pull/6264>#6264</a></li></ul><h2 id=1180-2024-09-03>1.18.0 2024-09-03</h2><ul><li>[CHANGE] Ingester: Remove <code>-querier.query-store-for-labels-enabled</code> flag. Querying long-term store for labels is always enabled. <a href=https://github.com/cortexproject/cortex/pull/5984>#5984</a></li><li>[CHANGE] Server: Instrument <code>cortex_request_duration_seconds</code> metric with native histogram. If <code>native-histograms</code> feature is enabled in monitoring Prometheus then the metric name needs to be updated in your dashboards. <a href=https://github.com/cortexproject/cortex/pull/6056>#6056</a></li><li>[CHANGE] Distributor/Ingester: Change <code>cortex_distributor_ingester_appends_total</code>, <code>cortex_distributor_ingester_append_failures_total</code>, <code>cortex_distributor_ingester_queries_total</code>, and <code>cortex_distributor_ingester_query_failures_total</code> metrics to use the ingester ID instead of its IP as the label value. <a href=https://github.com/cortexproject/cortex/pull/6078>#6078</a></li><li>[CHANGE] OTLP: Set <code>AddMetricSuffixes</code> to true to always enable metric name normalization. <a href=https://github.com/cortexproject/cortex/pull/6136>#6136</a></li><li>[CHANGE] Querier: Deprecate and enable by default <code>querier.ingester-metadata-streaming</code> flag. <a href=https://github.com/cortexproject/cortex/pull/6147>#6147</a></li><li>[CHANGE] QueryFrontend/QueryScheduler: Deprecate <code>-querier.max-outstanding-requests-per-tenant</code> and <code>-query-scheduler.max-outstanding-requests-per-tenant</code> flags. Use frontend.max-outstanding-requests-per-tenant instead. <a href=https://github.com/cortexproject/cortex/pull/6146>#6146</a></li><li>[CHANGE] Ingesters: Enable &lsquo;snappy-block&rsquo; compression on ingester clients by default. <a href=https://github.com/cortexproject/cortex/pull/6148>#6148</a></li><li>[CHANGE] Ruler: Scheduling <code>ruler.evaluation-delay-duration</code> to be deprecated. Ruler will use the highest value between <code>ruler.evaluation-delay-duration</code> and <code>ruler.query-offset</code> <a href=https://github.com/cortexproject/cortex/pull/6149>#6149</a></li><li>[CHANGE] Querier: Remove <code>-querier.at-modifier-enabled</code> flag. <a href=https://github.com/cortexproject/cortex/pull/6157>#6157</a></li><li>[CHANGE] Tracing: Remove deprecated <code>oltp_endpoint</code> config entirely. <a href=https://github.com/cortexproject/cortex/pull/6158>#6158</a></li><li>[CHANGE] Store Gateway: Enable store gateway zone stable shuffle sharding by default. <a href=https://github.com/cortexproject/cortex/pull/6161>#6161</a></li><li>[FEATURE] Ingester/Distributor: Experimental: Enable native histogram ingestion via <code>-blocks-storage.tsdb.enable-native-histograms</code> flag. <a href=https://github.com/cortexproject/cortex/pull/5986>#5986</a> <a href=https://github.com/cortexproject/cortex/pull/6010>#6010</a> <a href=https://github.com/cortexproject/cortex/pull/6020>#6020</a></li><li>[FEATURE] Querier: Enable querying native histogram chunks. <a href=https://github.com/cortexproject/cortex/pull/5944>#5944</a> <a href=https://github.com/cortexproject/cortex/pull/6031>#6031</a></li><li>[FEATURE] Query Frontend: Support native histogram in query frontend response. <a href=https://github.com/cortexproject/cortex/pull/5996>#5996</a> <a href=https://github.com/cortexproject/cortex/pull/6043>#6043</a></li><li>[FEATURE] Ruler: Support sending native histogram samples to Ingester. <a href=https://github.com/cortexproject/cortex/pull/6029>#6029</a></li><li>[FEATURE] Ruler: Add support for filtering out alerts in ListRules API. <a href=https://github.com/cortexproject/cortex/pull/6011>#6011</a></li><li>[FEATURE] Query Frontend: Added a query rejection mechanism to block resource-intensive queries. <a href=https://github.com/cortexproject/cortex/pull/6005>#6005</a></li><li>[FEATURE] OTLP: Support ingesting OTLP exponential metrics as native histograms. <a href=https://github.com/cortexproject/cortex/pull/6071>#6071</a> <a href=https://github.com/cortexproject/cortex/pull/6135>#6135</a></li><li>[FEATURE] Ingester: Add <code>ingester.instance-limits.max-inflight-query-requests</code> to allow limiting ingester concurrent queries. <a href=https://github.com/cortexproject/cortex/pull/6081>#6081</a></li><li>[FEATURE] Distributor: Add <code>validation.max-native-histogram-buckets</code> to limit max number of bucket count. Distributor will try to automatically reduce histogram resolution until it is within the bucket limit or resolution cannot be reduced anymore. <a href=https://github.com/cortexproject/cortex/pull/6104>#6104</a></li><li>[FEATURE] Store Gateway: Introduce token bucket limiter to enhance store gateway throttling. <a href=https://github.com/cortexproject/cortex/pull/6016>#6016</a></li><li>[FEATURE] Ruler: Add support for <code>query_offset</code> field on RuleGroup and new <code>ruler_query_offset</code> per-tenant limit. <a href=https://github.com/cortexproject/cortex/pull/6085>#6085</a></li><li>[ENHANCEMENT] Ruler: Add support to persist tokens in rulers. <a href=https://github.com/cortexproject/cortex/pull/5987>#5987</a></li><li>[ENHANCEMENT] Query Frontend/Querier: Added store gateway postings touched count and touched size in Querier stats and log in Query Frontend. <a href=https://github.com/cortexproject/cortex/pull/5892>#5892</a></li><li>[ENHANCEMENT] Query Frontend/Querier: Returns <code>warnings</code> on prometheus query responses. <a href=https://github.com/cortexproject/cortex/pull/5916>#5916</a></li><li>[ENHANCEMENT] Ingester: Allowing to configure <code>-blocks-storage.tsdb.head-compaction-interval</code> flag up to 30 min and add a jitter on the first head compaction. <a href=https://github.com/cortexproject/cortex/pull/5919>#5919</a> <a href=https://github.com/cortexproject/cortex/pull/5928>#5928</a></li><li>[ENHANCEMENT] Distributor: Added <code>max_inflight_push_requests</code> config to ingester client to protect distributor from OOMKilled. <a href=https://github.com/cortexproject/cortex/pull/5917>#5917</a></li><li>[ENHANCEMENT] Distributor/Querier: Clean stale per-ingester metrics after ingester restarts. <a href=https://github.com/cortexproject/cortex/pull/5930>#5930</a></li><li>[ENHANCEMENT] Distributor/Ring: Allow disabling detailed ring metrics by ring member. <a href=https://github.com/cortexproject/cortex/pull/5931>#5931</a></li><li>[ENHANCEMENT] KV: Etcd Added etcd.ping-without-stream-allowed parameter to disable/enable PermitWithoutStream <a href=https://github.com/cortexproject/cortex/pull/5933>#5933</a></li><li>[ENHANCEMENT] Ingester: Add a new <code>limits_per_label_set</code> limit. This limit functions similarly to <code>max_series_per_metric</code>, but allowing users to define the maximum number of series per LabelSet. <a href=https://github.com/cortexproject/cortex/pull/5950>#5950</a> <a href=https://github.com/cortexproject/cortex/pull/5993>#5993</a></li><li>[ENHANCEMENT] Store Gateway: Log gRPC requests together with headers configured in <code>http_request_headers_to_log</code>. <a href=https://github.com/cortexproject/cortex/pull/5958>#5958</a></li><li>[ENHANCEMENT] Ingester: Add a new experimental <code>-ingester.labels-string-interning-enabled</code> flag to enable string interning for metrics labels. <a href=https://github.com/cortexproject/cortex/pull/6057>#6057</a></li><li>[ENHANCEMENT] Ingester: Add link to renew 10% of the ingesters tokens in the admin page. <a href=https://github.com/cortexproject/cortex/pull/6063>#6063</a></li><li>[ENHANCEMENT] Ruler: Add support for filtering by <code>state</code> and <code>health</code> field on Rules API. <a href=https://github.com/cortexproject/cortex/pull/6040>#6040</a></li><li>[ENHANCEMENT] Ruler: Add support for filtering by <code>match</code> field on Rules API. <a href=https://github.com/cortexproject/cortex/pull/6083>#6083</a></li><li>[ENHANCEMENT] Distributor: Reduce memory usage when error volume is high. <a href=https://github.com/cortexproject/cortex/pull/6095>#6095</a></li><li>[ENHANCEMENT] Compactor: Centralize metrics used by compactor and add user label to compactor metrics. <a href=https://github.com/cortexproject/cortex/pull/6096>#6096</a></li><li>[ENHANCEMENT] Compactor: Add unique execution ID for each compaction cycle in log for easy debugging. <a href=https://github.com/cortexproject/cortex/pull/6097>#6097</a></li><li>[ENHANCEMENT] Compactor: Differentiate retry and halt error and retry failed compaction only on retriable error. <a href=https://github.com/cortexproject/cortex/pull/6111>#6111</a></li><li>[ENHANCEMENT] Ruler: Add support for filtering by <code>state</code> and <code>health</code> field on Rules API. <a href=https://github.com/cortexproject/cortex/pull/6040>#6040</a></li><li>[ENHANCEMENT] Compactor: Split cleaner cycle for active and deleted tenants. <a href=https://github.com/cortexproject/cortex/pull/6112>#6112</a></li><li>[ENHANCEMENT] Compactor: Introduce cleaner visit marker. <a href=https://github.com/cortexproject/cortex/pull/6113>#6113</a></li><li>[ENHANCEMENT] Query Frontend: Add <code>cortex_query_samples_total</code> metric. <a href=https://github.com/cortexproject/cortex/pull/6142>#6142</a></li><li>[ENHANCEMENT] Ingester: Implement metadata API limit. <a href=https://github.com/cortexproject/cortex/pull/6128>#6128</a></li><li>[BUGFIX] Configsdb: Fix endline issue in db password. <a href=https://github.com/cortexproject/cortex/pull/5920>#5920</a></li><li>[BUGFIX] Ingester: Fix <code>user</code> and <code>type</code> labels for the <code>cortex_ingester_tsdb_head_samples_appended_total</code> TSDB metric. <a href=https://github.com/cortexproject/cortex/pull/5952>#5952</a></li><li>[BUGFIX] Querier: Enforce max query length check for <code>/api/v1/series</code> API even though <code>ignoreMaxQueryLength</code> is set to true. <a href=https://github.com/cortexproject/cortex/pull/6018>#6018</a></li><li>[BUGFIX] Ingester: Fix issue with the minimize token generator where it was not taking in consideration the current ownership of an instance when generating extra tokens. <a href=https://github.com/cortexproject/cortex/pull/6062>#6062</a></li><li>[BUGFIX] Scheduler: Fix user queue in scheduler that was not thread-safe. <a href=https://github.com/cortexproject/cortex/pull/6077>#6077</a> <a href=https://github.com/cortexproject/cortex/pull/6160>#6160</a></li><li>[BUGFIX] Ingester: Include out-of-order head compaction when compacting TSDB head. <a href=https://github.com/cortexproject/cortex/pull/6108>#6108</a></li><li>[BUGFIX] Ingester: Fix <code>cortex_ingester_tsdb_mmap_chunks_total</code> metric. <a href=https://github.com/cortexproject/cortex/pull/6134>#6134</a></li><li>[BUGFIX] Query Frontend: Fix query rejection bug for metadata queries. <a href=https://github.com/cortexproject/cortex/pull/6143>#6143</a></li></ul><h2 id=1171-2024-05-20>1.17.1 2024-05-20</h2><ul><li>[CHANGE] Query Frontend/Ruler: Omit empty data, errorType and error fields in API response. <a href=https://github.com/cortexproject/cortex/pull/5953>#5953</a> <a href=https://github.com/cortexproject/cortex/pull/5954>#5954</a></li><li>[ENHANCEMENT] Ingester: Added <code>upload_compacted_blocks_enabled</code> config to ingester to parameterize uploading compacted blocks. <a href=https://github.com/cortexproject/cortex/pull/5959>#5959</a></li><li>[BUGFIX] Querier: Select correct tenant during query federation. <a href=https://github.com/cortexproject/cortex/pull/5943>#5943</a></li></ul><h2 id=1170-2024-04-30>1.17.0 2024-04-30</h2><ul><li>[CHANGE] Azure Storage: Upgraded objstore dependency and support Azure Workload Identity Authentication. Added <code>connection_string</code> to support authenticating via SAS token. Marked <code>msi_resource</code> config as deprecating. <a href=https://github.com/cortexproject/cortex/pull/5645>#5645</a></li><li>[CHANGE] Store Gateway: Add a new fastcache based inmemory index cache. <a href=https://github.com/cortexproject/cortex/pull/5619>#5619</a></li><li>[CHANGE] Index Cache: Multi level cache backfilling operation becomes async. Added <code>-blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency</code> and <code>-blocks-storage.bucket-store.index-cache.multilevel.max-async-buffer-size</code> configs and metric <code>cortex_store_multilevel_index_cache_backfill_dropped_items_total</code> for number of dropped items. <a href=https://github.com/cortexproject/cortex/pull/5661>#5661</a></li><li>[CHANGE] Ingester: Disable uploading compacted blocks and overlapping compaction in ingester. <a href=https://github.com/cortexproject/cortex/pull/5735>#5735</a></li><li>[CHANGE] Distributor: Count the number of rate-limited samples in <code>distributor_samples_in_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5714>#5714</a></li><li>[CHANGE] Ruler: Remove <code>cortex_ruler_write_requests_total</code>, <code>cortex_ruler_write_requests_failed_total</code>, <code>cortex_ruler_queries_total</code>, <code>cortex_ruler_queries_failed_total</code>, and <code>cortex_ruler_query_seconds_total</code> metrics for the tenant when the ruler deletes the manager for the tenant. <a href=https://github.com/cortexproject/cortex/pull/5772>#5772</a></li><li>[CHANGE] Main: Mark <code>mem-ballast-size-bytes</code> flag as deprecated. <a href=https://github.com/cortexproject/cortex/pull/5816>#5816</a></li><li>[CHANGE] Querier: Mark <code>-querier.ingester-streaming</code> flag as deprecated. Now query ingester streaming is always enabled. <a href=https://github.com/cortexproject/cortex/pull/5817>#5817</a></li><li>[CHANGE] Compactor/Bucket Store: Added <code>-blocks-storage.bucket-store.block-discovery-strategy</code> to configure different block listing strategy. Reverted the current recursive block listing mechanism and use the strategy <code>Concurrent</code> as in 1.15. <a href=https://github.com/cortexproject/cortex/pull/5828>#5828</a></li><li>[CHANGE] Compactor: Don&rsquo;t halt compactor when overlapped source blocks detected. <a href=https://github.com/cortexproject/cortex/pull/5854>#5854</a></li><li>[CHANGE] S3 Bucket Client: Expose <code>-blocks-storage.s3.send-content-md5</code> flag and set default checksum algorithm to MD5. <a href=https://github.com/cortexproject/cortex/pull/5870>#5870</a></li><li>[CHANGE] Querier: Mark <code>querier.iterators</code> and <code>querier.batch-iterators</code> flags as deprecated. Now querier always use batch iterators. <a href=https://github.com/cortexproject/cortex/pull/5868>#5868</a></li><li>[CHANGE] Query Frontend: Error response returned by Query Frontend now follows Prometheus API error response format. <a href=https://github.com/cortexproject/cortex/pull/5811>#5811</a></li><li>[FEATURE] Experimental: OTLP ingestion. <a href=https://github.com/cortexproject/cortex/pull/5813>#5813</a></li><li>[FEATURE] Query Frontend/Scheduler: Add query priority support. <a href=https://github.com/cortexproject/cortex/pull/5605>#5605</a></li><li>[FEATURE] Tracing: Use <code>kuberesolver</code> to resolve OTLP endpoints address with <code>kubernetes://</code> prefix as Kubernetes service. <a href=https://github.com/cortexproject/cortex/pull/5731>#5731</a></li><li>[FEATURE] Tracing: Add <code>tracing.otel.round-robin</code> flag to use <code>round_robin</code> gRPC client side LB policy for sending OTLP traces. <a href=https://github.com/cortexproject/cortex/pull/5731>#5731</a></li><li>[FEATURE] Ruler: Add <code>ruler.concurrent-evals-enabled</code> flag to enable concurrent evaluation within a single rule group for independent rules. Maximum concurrency can be configured via <code>ruler.max-concurrent-evals</code>. <a href=https://github.com/cortexproject/cortex/pull/5766>#5766</a></li><li>[FEATURE] Distributor Queryable: Experimental: Add config <code>zone_results_quorum_metadata</code>. When querying ingesters using metadata APIs such as label names and values, only results from quorum number of zones will be included and merged. <a href=https://github.com/cortexproject/cortex/pull/5779>#5779</a></li><li>[FEATURE] Storage Cache Clients: Add config <code>set_async_circuit_breaker_config</code> to utilize the circuit breaker pattern for dynamically thresholding asynchronous set operations. Implemented in both memcached and redis cache clients. <a href=https://github.com/cortexproject/cortex/pull/5789>#5789</a></li><li>[FEATURE] Ruler: Add experimental <code>experimental.ruler.api-deduplicate-rules</code> flag to remove duplicate rule groups from the Prometheus compatible rules API endpoint. Add experimental <code>ruler.ring.replication-factor</code> and <code>ruler.ring.zone-awareness-enabled</code> flags to configure rule group replication, but only the first ruler in the replicaset evaluates the rule group, the rest will be used as backup to handle events when a ruler is down during an API request to list rules. <a href=https://github.com/cortexproject/cortex/pull/5782>#5782</a> <a href=https://github.com/cortexproject/cortex/pull/5901>#5901</a></li><li>[FEATURE] Ring: Add experimental <code>-ingester.tokens-generator-strategy=minimize-spread</code> flag to enable the new minimize spread token generator strategy. <a href=https://github.com/cortexproject/cortex/pull/5855>#5855</a></li><li>[FEATURE] Ring Status Page: Add <code>Ownership Diff From Expected</code> column in the ring table to indicate the extent to which the ownership of a specific ingester differs from the expected ownership. <a href=https://github.com/cortexproject/cortex/pull/5889>#5889</a></li><li>[ENHANCEMENT] Ingester: Add per-tenant new metric <code>cortex_ingester_tsdb_data_replay_duration_seconds</code>. <a href=https://github.com/cortexproject/cortex/pull/5477>#5477</a></li><li>[ENHANCEMENT] Store Gateway: Added <code>-store-gateway.enabled-tenants</code> and <code>-store-gateway.disabled-tenants</code> to explicitly enable or disable store-gateway for specific tenants. <a href=https://github.com/cortexproject/cortex/pull/5638>#5638</a></li><li>[ENHANCEMENT] Query Frontend: Write service timing header in response even though there is an error. <a href=https://github.com/cortexproject/cortex/pull/5653>#5653</a></li><li>[ENHANCEMENT] Compactor: Add new compactor metric <code>cortex_compactor_start_duration_seconds</code>. <a href=https://github.com/cortexproject/cortex/pull/5683>#5683</a></li><li>[ENHANCEMENT] Index Cache: Multi level cache adds config <code>max_backfill_items</code> to cap max items to backfill per async operation. <a href=https://github.com/cortexproject/cortex/pull/5686>#5686</a></li><li>[ENHANCEMENT] Query Frontend: Log number of split queries in <code>query stats</code> log. <a href=https://github.com/cortexproject/cortex/pull/5703>#5703</a></li><li>[ENHANCEMENT] Compactor: Skip compaction retry when encountering a permission denied error. <a href=https://github.com/cortexproject/cortex/pull/5727>#5727</a></li><li>[ENHANCEMENT] Logging: Added new options for logging HTTP request headers: <code>-server.log-request-headers</code> enables logging HTTP request headers, <code>-server.log-request-headers-exclude-list</code> allows users to specify headers which should not be logged. <a href=https://github.com/cortexproject/cortex/pull/5744>#5744</a></li><li>[ENHANCEMENT] Query Frontend/Scheduler: Time check in query priority now considers overall data select time window (including range selectors, modifiers and lookback delta). <a href=https://github.com/cortexproject/cortex/pull/5758>#5758</a></li><li>[ENHANCEMENT] Querier: Added <code>querier.store-gateway-query-stats-enabled</code> to enable or disable store gateway query stats log. <a href=https://github.com/cortexproject/cortex/pull/5749>#5749</a></li><li>[ENHANCEMENT] Querier: Improve labels APIs latency by merging slices using K-way merge and more than 1 core. <a href=https://github.com/cortexproject/cortex/pull/5785>#5785</a></li><li>[ENHANCEMENT] AlertManager: Retrying AlertManager Delete Silence on error. <a href=https://github.com/cortexproject/cortex/pull/5794>#5794</a></li><li>[ENHANCEMENT] Ingester: Add new ingester metric <code>cortex_ingester_max_inflight_query_requests</code>. <a href=https://github.com/cortexproject/cortex/pull/5798>#5798</a></li><li>[ENHANCEMENT] Query: Added <code>query_storage_wall_time</code> to Query Frontend and Ruler query stats log for wall time spent on fetching data from storage. Query evaluation is not included. <a href=https://github.com/cortexproject/cortex/pull/5799>#5799</a></li><li>[ENHANCEMENT] Query: Added additional max query length check at Query Frontend and Ruler. Added <code>-querier.ignore-max-query-length</code> flag to disable max query length check at Querier. <a href=https://github.com/cortexproject/cortex/pull/5808>#5808</a></li><li>[ENHANCEMENT] Querier: Add context error check when converting Metrics to SeriesSet for GetSeries on distributorQuerier. <a href=https://github.com/cortexproject/cortex/pull/5827>#5827</a></li><li>[ENHANCEMENT] Ruler: Improve GetRules response time by reducing lock contention and introducing a temporary rules cache in <code>ruler/manager.go</code>. <a href=https://github.com/cortexproject/cortex/pull/5805>#5805</a></li><li>[ENHANCEMENT] Querier: Add context error check when merging slices from ingesters for GetLabel operations. <a href=https://github.com/cortexproject/cortex/pull/5837>#5837</a></li><li>[BUGFIX] Distributor: Do not use label with empty values for sharding <a href=https://github.com/cortexproject/cortex/pull/5717>#5717</a></li><li>[BUGFIX] Query Frontend: queries with negative offset should check whether it is cacheable or not. <a href=https://github.com/cortexproject/cortex/pull/5719>#5719</a></li><li>[BUGFIX] Redis Cache: pass <code>cache_size</code> config correctly. <a href=https://github.com/cortexproject/cortex/pull/5734>#5734</a></li><li>[BUGFIX] Distributor: Shuffle-Sharding with <code>ingestion_tenant_shard_size</code> set to 0, default sharding strategy should be used. <a href=https://github.com/cortexproject/cortex/pull/5189>#5189</a></li><li>[BUGFIX] Cortex: Fix GRPC stream clients not honoring overrides for call options. <a href=https://github.com/cortexproject/cortex/pull/5797>#5797</a></li><li>[BUGFIX] Ruler: Fix support for <code>keep_firing_for</code> field in alert rules. <a href=https://github.com/cortexproject/cortex/pull/5823>#5823</a></li><li>[BUGFIX] Ring DDB: Fix lifecycle for ring counting unhealthy pods as healthy. <a href=https://github.com/cortexproject/cortex/pull/5838>#5838</a></li><li>[BUGFIX] Ring DDB: Fix region assignment. <a href=https://github.com/cortexproject/cortex/pull/5842>#5842</a></li></ul><h2 id=1161-2024-04-23>1.16.1 2024-04-23</h2><ul><li>[ENHANCEMENT] Upgraded Docker base images to <code>alpine:3.18</code>. <a href=https://github.com/cortexproject/cortex/pull/5684>#5684</a></li><li>[ENHANCEMENT] Upgrade to go 1.21.9 <a href=https://github.com/cortexproject/cortex/pull/5879>#5879</a> <a href=https://github.com/cortexproject/cortex/pull/5882>#5882</a></li></ul><h2 id=1160-2023-11-20>1.16.0 2023-11-20</h2><ul><li>[CHANGE] AlertManager: include reason label in <code>cortex_alertmanager_notifications_failed_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5409>#5409</a></li><li>[CHANGE] Ruler: Added user label to <code>cortex_ruler_write_requests_total</code>, <code>cortex_ruler_write_requests_failed_total</code>, <code>cortex_ruler_queries_total</code>, and <code>cortex_ruler_queries_failed_total</code> metrics. <a href=https://github.com/cortexproject/cortex/pull/5312>#5312</a></li><li>[CHANGE] Alertmanager: Validating new fields on the PagerDuty AM config. <a href=https://github.com/cortexproject/cortex/pull/5290>#5290</a></li><li>[CHANGE] Ingester: Creating label <code>native-histogram-sample</code> on the <code>cortex_discarded_samples_total</code> to keep track of discarded native histogram samples. <a href=https://github.com/cortexproject/cortex/pull/5289>#5289</a></li><li>[CHANGE] Store Gateway: Rename <code>cortex_bucket_store_cached_postings_compression_time_seconds</code> to <code>cortex_bucket_store_cached_postings_compression_time_seconds_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5431>#5431</a></li><li>[CHANGE] Store Gateway: Rename <code>cortex_bucket_store_cached_series_fetch_duration_seconds</code> to <code>cortex_bucket_store_series_fetch_duration_seconds</code> and <code>cortex_bucket_store_cached_postings_fetch_duration_seconds</code> to <code>cortex_bucket_store_postings_fetch_duration_seconds</code>. Add new metric <code>cortex_bucket_store_chunks_fetch_duration_seconds</code>. <a href=https://github.com/cortexproject/cortex/pull/5448>#5448</a></li><li>[CHANGE] Store Gateway: Remove <code>idle_timeout</code>, <code>max_conn_age</code>, <code>pool_size</code>, <code>min_idle_conns</code> fields for Redis index cache and caching bucket. <a href=https://github.com/cortexproject/cortex/pull/5448>#5448</a></li><li>[CHANGE] Store Gateway: Add flag <code>-store-gateway.sharding-ring.zone-stable-shuffle-sharding</code> to enable store gateway to use zone stable shuffle sharding. <a href=https://github.com/cortexproject/cortex/pull/5489>#5489</a></li><li>[CHANGE] Bucket Index: Add <code>series_max_size</code> and <code>chunk_max_size</code> to bucket index. <a href=https://github.com/cortexproject/cortex/pull/5489>#5489</a></li><li>[CHANGE] StoreGateway: Rename <code>cortex_bucket_store_chunk_pool_returned_bytes_total</code> and <code>cortex_bucket_store_chunk_pool_requested_bytes_total</code> to <code>cortex_bucket_store_chunk_pool_operation_bytes_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5552>#5552</a></li><li>[CHANGE] Query Frontend/Querier: Make build info API disabled by default and add feature flag <code>api.build-info-enabled</code> to enable it. <a href=https://github.com/cortexproject/cortex/pull/5533>#5533</a></li><li>[CHANGE] Purger: Do no use S3 tenant kms key when uploading deletion marker. <a href=https://github.com/cortexproject/cortex/pull/5575>#5575</a></li><li>[CHANGE] Ingester: Shipper always allows uploading compacted blocks to ship OOO compacted blocks. <a href=https://github.com/cortexproject/cortex/pull/5625>#5625</a></li><li>[CHANGE] DDBKV: Change metric name from <code>dynamodb_kv_read_capacity_total</code> to <code>dynamodb_kv_consumed_capacity_total</code> and include Delete, Put, Batch dimension. <a href=https://github.com/cortexproject/cortex/pull/5487>#5487</a></li><li>[CHANGE] Compactor: Adding the userId on the compact dir path. <a href=https://github.com/cortexproject/cortex/pull/5524>#5524</a></li><li>[CHANGE] Ingester: Remove deprecated ingester metrics. <a href=https://github.com/cortexproject/cortex/pull/5472>#5472</a></li><li>[CHANGE] Query Frontend: Expose <code>-querier.max-subquery-steps</code> to configure subquery max steps check. By default, the limit is set to 0, which is disabled. <a href=https://github.com/cortexproject/cortex/pull/5656>#5656</a></li><li>[FEATURE] Store Gateway: Implementing multi level index cache. <a href=https://github.com/cortexproject/cortex/pull/5451>#5451</a></li><li>[FEATURE] Ruler: Add support for disabling rule groups. <a href=https://github.com/cortexproject/cortex/pull/5521>#5521</a></li><li>[FEATURE] Support object storage backends for runtime configuration file. <a href=https://github.com/cortexproject/cortex/pull/5292>#5292</a></li><li>[FEATURE] Ruler: Add support for <code>Limit</code> field on RuleGroup. <a href=https://github.com/cortexproject/cortex/pull/5528>#5528</a></li><li>[FEATURE] AlertManager: Add support for Webex, Discord and Telegram Receiver. <a href=https://github.com/cortexproject/cortex/pull/5493>#5493</a></li><li>[FEATURE] Ingester: added <code>-admin-limit-message</code> to customize the message contained in limit errors.<a href=https://github.com/cortexproject/cortex/pull/5460>#5460</a></li><li>[FEATURE] AlertManager: Update version to v0.26.0 and bring in Microsoft Teams receiver. <a href=https://github.com/cortexproject/cortex/pull/5543>#5543</a></li><li>[FEATURE] Store Gateway: Support lazy expanded posting optimization. Added new flag <code>blocks-storage.bucket-store.lazy-expanded-postings-enabled</code> and new metrics <code>cortex_bucket_store_lazy_expanded_postings_total</code>, <code>cortex_bucket_store_lazy_expanded_posting_size_bytes_total</code> and <code>cortex_bucket_store_lazy_expanded_posting_series_overfetched_size_bytes_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5556>#5556</a>.</li><li>[FEATURE] Store Gateway: Add <code>max_downloaded_bytes_per_request</code> to limit max bytes to download per store gateway request. <a href=https://github.com/cortexproject/cortex/pull/5179>#5179</a></li><li>[FEATURE] Added 2 flags <code>-alertmanager.alertmanager-client.grpc-max-send-msg-size</code> and <code>-alertmanager.alertmanager-client.grpc-max-recv-msg-size</code> to configure alert manager grpc client message size limits. <a href=https://github.com/cortexproject/cortex/pull/5338>#5338</a></li><li>[FEATURE] Querier/StoreGateway: Allow the tenant shard sizes to be a percent of total instances. <a href=https://github.com/cortexproject/cortex/pull/5393>#5393</a></li><li>[FEATURE] Added the flag <code>-alertmanager.api-concurrency</code> to configure alert manager api concurrency limit. <a href=https://github.com/cortexproject/cortex/pull/5412>#5412</a></li><li>[FEATURE] Store Gateway: Add <code>-store-gateway.sharding-ring.keep-instance-in-the-ring-on-shutdown</code> to skip unregistering instance from the ring in shutdown. <a href=https://github.com/cortexproject/cortex/pull/5421>#5421</a></li><li>[FEATURE] Ruler: Support for filtering rules in the API. <a href=https://github.com/cortexproject/cortex/pull/5417>#5417</a></li><li>[FEATURE] Compactor: Add <code>-compactor.ring.tokens-file-path</code> to store generated tokens locally. <a href=https://github.com/cortexproject/cortex/pull/5432>#5432</a></li><li>[FEATURE] Query Frontend: Add <code>-frontend.retry-on-too-many-outstanding-requests</code> to re-enqueue 429 requests if there are multiple query-schedulers available. <a href=https://github.com/cortexproject/cortex/pull/5496>#5496</a></li><li>[FEATURE] Store Gateway: Add <code>-blocks-storage.bucket-store.max-inflight-requests</code> for store gateways to reject further series requests upon reaching the limit. <a href=https://github.com/cortexproject/cortex/pull/5553>#5553</a></li><li>[FEATURE] Store Gateway: Support filtered index cache. <a href=https://github.com/cortexproject/cortex/pull/5587>#5587</a></li><li>[ENHANCEMENT] Update go version to 1.21.3. <a href=https://github.com/cortexproject/cortex/pull/5630>#5630</a></li><li>[ENHANCEMENT] Store Gateway: Add <code>cortex_bucket_store_block_load_duration_seconds</code> histogram to track time to load blocks. <a href=https://github.com/cortexproject/cortex/pull/5580>#5580</a></li><li>[ENHANCEMENT] Querier: retry chunk pool exhaustion error in querier rather than query frontend. <a href=https://github.com/cortexproject/cortex/pull/5569>#5569</a></li><li>[ENHANCEMENT] Alertmanager: Added flag <code>-alertmanager.alerts-gc-interval</code> to configure alerts Garbage collection interval. <a href=https://github.com/cortexproject/cortex/pull/5550>#5550</a></li><li>[ENHANCEMENT] Query Frontend: enable vertical sharding on binary expr . <a href=https://github.com/cortexproject/cortex/pull/5507>#5507</a></li><li>[ENHANCEMENT] Query Frontend: Include user agent as part of query frontend log. <a href=https://github.com/cortexproject/cortex/pull/5450>#5450</a></li><li>[ENHANCEMENT] Query: Set CORS Origin headers for Query API <a href=https://github.com/cortexproject/cortex/pull/5388>#5388</a></li><li>[ENHANCEMENT] Query Frontend: Add <code>cortex_rejected_queries_total</code> metric for throttled queries. <a href=https://github.com/cortexproject/cortex/pull/5356>#5356</a></li><li>[ENHANCEMENT] Query Frontend: Optimize the decoding of <code>SampleStream</code>. <a href=https://github.com/cortexproject/cortex/pull/5349>#5349</a></li><li>[ENHANCEMENT] Compactor: Check ctx done when uploading visit marker. <a href=https://github.com/cortexproject/cortex/pull/5333>#5333</a></li><li>[ENHANCEMENT] AlertManager: Add <code>cortex_alertmanager_dispatcher_aggregation_groups</code> and <code>cortex_alertmanager_dispatcher_alert_processing_duration_seconds</code> metrics for dispatcher. <a href=https://github.com/cortexproject/cortex/pull/5592>#5592</a></li><li>[ENHANCEMENT] Store Gateway: Added new flag <code>blocks-storage.bucket-store.series-batch-size</code> to control how many series to fetch per batch in Store Gateway. <a href=https://github.com/cortexproject/cortex/pull/5582>#5582</a>.</li><li>[ENHANCEMENT] Querier: Log query stats when querying store gateway. <a href=https://github.com/cortexproject/cortex/pull/5376>#5376</a></li><li>[ENHANCEMENT] Ruler: Add <code>cortex_ruler_rule_group_load_duration_seconds</code> and <code>cortex_ruler_rule_group_sync_duration_seconds</code> metrics. <a href=https://github.com/cortexproject/cortex/pull/5609>#5609</a></li><li>[ENHANCEMENT] Ruler: Add contextual info and query statistics to log <a href=https://github.com/cortexproject/cortex/pull/5604>#5604</a></li><li>[ENHANCEMENT] Distributor/Ingester: Add span on push path <a href=https://github.com/cortexproject/cortex/pull/5319>#5319</a></li><li>[ENHANCEMENT] Query Frontend: Reject subquery with too small step size. <a href=https://github.com/cortexproject/cortex/pull/5323>#5323</a></li><li>[ENHANCEMENT] Compactor: Exposing Thanos <code>accept-malformed-index</code> to Cortex compactor. <a href=https://github.com/cortexproject/cortex/pull/5334>#5334</a></li><li>[ENHANCEMENT] Log: Avoid expensive <code>log.Valuer</code> evaluation for disallowed levels. <a href=https://github.com/cortexproject/cortex/pull/5297>#5297</a></li><li>[ENHANCEMENT] Improving Performance on the API Gzip Handler. <a href=https://github.com/cortexproject/cortex/pull/5347>#5347</a></li><li>[ENHANCEMENT] Dynamodb: Add <code>puller-sync-time</code> to allow different pull time for ring. <a href=https://github.com/cortexproject/cortex/pull/5357>#5357</a></li><li>[ENHANCEMENT] Emit querier <code>max_concurrent</code> as a metric. <a href=https://github.com/cortexproject/cortex/pull/5362>#5362</a></li><li>[ENHANCEMENT] Avoid sort tokens on lifecycler autoJoin. <a href=https://github.com/cortexproject/cortex/pull/5394>#5394</a></li><li>[ENHANCEMENT] Do not resync blocks in running store gateways during rollout deployment and container restart. <a href=https://github.com/cortexproject/cortex/pull/5363>#5363</a></li><li>[ENHANCEMENT] Store Gateway: Add new metrics <code>cortex_bucket_store_sent_chunk_size_bytes</code>, <code>cortex_bucket_store_postings_size_bytes</code> and <code>cortex_bucket_store_empty_postings_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5397>#5397</a></li><li>[ENHANCEMENT] Add jitter to lifecycler heartbeat. <a href=https://github.com/cortexproject/cortex/pull/5404>#5404</a></li><li>[ENHANCEMENT] Store Gateway: Add config <code>estimated_max_series_size_bytes</code> and <code>estimated_max_chunk_size_bytes</code> to address data overfetch. <a href=https://github.com/cortexproject/cortex/pull/5401>#5401</a></li><li>[ENHANCEMENT] Distributor/Ingester: Add experimental <code>-distributor.sign_write_requests</code> flag to sign the write requests. <a href=https://github.com/cortexproject/cortex/pull/5430>#5430</a></li><li>[ENHANCEMENT] Store Gateway/Querier/Compactor: Handling CMK Access Denied errors. <a href=https://github.com/cortexproject/cortex/pull/5420>#5420</a> <a href=https://github.com/cortexproject/cortex/pull/5442>#5442</a> <a href=https://github.com/cortexproject/cortex/pull/5446>#5446</a></li><li>[ENHANCEMENT] Alertmanager: Add the alert name in error log when it get throttled. <a href=https://github.com/cortexproject/cortex/pull/5456>#5456</a></li><li>[ENHANCEMENT] Querier: Retry store gateway on different zones when zone awareness is enabled. <a href=https://github.com/cortexproject/cortex/pull/5476>#5476</a></li><li>[ENHANCEMENT] Compactor: allow <code>unregister_on_shutdown</code> to be configurable. <a href=https://github.com/cortexproject/cortex/pull/5503>#5503</a></li><li>[ENHANCEMENT] Querier: Batch adding series to query limiter to optimize locking. <a href=https://github.com/cortexproject/cortex/pull/5505>#5505</a></li><li>[ENHANCEMENT] Store Gateway: add metric <code>cortex_bucket_store_chunk_refetches_total</code> for number of chunk refetches. <a href=https://github.com/cortexproject/cortex/pull/5532>#5532</a></li><li>[ENHANCEMENT] BasicLifeCycler: allow final-sleep during shutdown <a href=https://github.com/cortexproject/cortex/pull/5517>#5517</a></li><li>[ENHANCEMENT] All: Handling CMK Access Denied errors. <a href=https://github.com/cortexproject/cortex/pull/5420>#5420</a> <a href=https://github.com/cortexproject/cortex/pull/5542>#5542</a></li><li>[ENHANCEMENT] Querier: Retry store gateway client connection closing gRPC error. <a href=https://github.com/cortexproject/cortex/pull/5558>#5558</a></li><li>[ENHANCEMENT] QueryFrontend: Add generic retry for all APIs. <a href=https://github.com/cortexproject/cortex/pull/5561>#5561</a>.</li><li>[ENHANCEMENT] Querier: Check context before notifying scheduler and frontend. <a href=https://github.com/cortexproject/cortex/pull/5565>#5565</a></li><li>[ENHANCEMENT] QueryFrontend: Add metric for number of series requests. <a href=https://github.com/cortexproject/cortex/pull/5373>#5373</a></li><li>[ENHANCEMENT] Store Gateway: Add histogram metrics for total time spent fetching series and chunks per request. <a href=https://github.com/cortexproject/cortex/pull/5573>#5573</a></li><li>[ENHANCEMENT] Store Gateway: Check context in multi level cache. Add <code>cortex_store_multilevel_index_cache_fetch_duration_seconds</code> and <code>cortex_store_multilevel_index_cache_backfill_duration_seconds</code> to measure fetch and backfill latency. <a href=https://github.com/cortexproject/cortex/pull/5596>#5596</a></li><li>[ENHANCEMENT] Ingester: Added new ingester TSDB metrics <code>cortex_ingester_tsdb_head_samples_appended_total</code>, <code>cortex_ingester_tsdb_head_out_of_order_samples_appended_total</code>, <code>cortex_ingester_tsdb_snapshot_replay_error_total</code>, <code>cortex_ingester_tsdb_sample_ooo_delta</code> and <code>cortex_ingester_tsdb_mmap_chunks_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5624>#5624</a></li><li>[ENHANCEMENT] Query Frontend: Handle context error before decoding and merging responses. <a href=https://github.com/cortexproject/cortex/pull/5499>#5499</a></li><li>[ENHANCEMENT] Store-Gateway and AlertManager: Add a <code>wait_instance_time_out</code> to context to avoid waiting forever. <a href=https://github.com/cortexproject/cortex/pull/5581>#5581</a></li><li>[ENHANCEMENT] Blocks storage: Move the tenant deletion mark from <code>&lt;tenantID>/markers/tenant-deletion-mark.json</code> to <code>__markers__/&lt;tenantID>/tenant-deletion-mark.json</code>. <a href=https://github.com/cortexproject/cortex/pull/5676>#5676</a></li><li>[BUGFIX] Compactor: Fix possible division by zero during compactor config validation. <a href=https://github.com/cortexproject/cortex/pull/5535>#5535</a></li><li>[BUGFIX] Ruler: Validate if rule group can be safely converted back to rule group yaml from protobuf message <a href=https://github.com/cortexproject/cortex/pull/5265>#5265</a></li><li>[BUGFIX] Querier: Convert gRPC <code>ResourceExhausted</code> status code from store gateway to 422 limit error. <a href=https://github.com/cortexproject/cortex/pull/5286>#5286</a></li><li>[BUGFIX] Alertmanager: Route web-ui requests to the alertmanager distributor when sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/5293>#5293</a></li><li>[BUGFIX] Storage: Bucket index updater should ignore meta not found for partial blocks. <a href=https://github.com/cortexproject/cortex/pull/5343>#5343</a></li><li>[BUGFIX] Ring: Add <code>JOINING</code> state to read operation. <a href=https://github.com/cortexproject/cortex/pull/5346>#5346</a></li><li>[BUGFIX] Compactor: Partial block with only visit marker should be deleted even there is no deletion marker. <a href=https://github.com/cortexproject/cortex/pull/5342>#5342</a></li><li>[BUGFIX] KV: Etcd calls will no longer block indefinitely and will now time out after the <code>DialTimeout</code> period. <a href=https://github.com/cortexproject/cortex/pull/5392>#5392</a></li><li>[BUGFIX] Ring: Allow RF greater than number of zones to select more than one instance per zone <a href=https://github.com/cortexproject/cortex/pull/5411>#5411</a></li><li>[BUGFIX] Store Gateway: Fix bug in store gateway ring comparison logic. <a href=https://github.com/cortexproject/cortex/pull/5426>#5426</a></li><li>[BUGFIX] Ring: Fix bug in consistency of Get func in a scaling zone-aware ring. <a href=https://github.com/cortexproject/cortex/pull/5429>#5429</a></li><li>[BUGFIX] Compactor: Fix retry on markers. <a href=https://github.com/cortexproject/cortex/pull/5441>#5441</a></li><li>[BUGFIX] Query Frontend: Fix bug of failing to cancel downstream request context in query frontend v2 mode (query scheduler enabled). <a href=https://github.com/cortexproject/cortex/pull/5447>#5447</a></li><li>[BUGFIX] Alertmanager: Remove the user id from state replication key metric label value. <a href=https://github.com/cortexproject/cortex/pull/5453>#5453</a></li><li>[BUGFIX] Compactor: Avoid cleaner concurrency issues checking global markers before all blocks. <a href=https://github.com/cortexproject/cortex/pull/5457>#5457</a></li><li>[BUGFIX] DDBKV: Disallow instance with older timestamp to update instance with newer timestamp. <a href=https://github.com/cortexproject/cortex/pull/5480>#5480</a></li><li>[BUGFIX] DDBKV: When no change detected in ring, retry the CAS until there is change. <a href=https://github.com/cortexproject/cortex/pull/5502>#5502</a></li><li>[BUGFIX] Fix bug on objstore when configured to use S3 fips endpoints. <a href=https://github.com/cortexproject/cortex/pull/5540>#5540</a></li><li>[BUGFIX] Ruler: Fix bug on ruler where a failure to load a single RuleGroup would prevent rulers to sync all RuleGroup. <a href=https://github.com/cortexproject/cortex/pull/5563>#5563</a></li><li>[BUGFIX] Query Frontend: Fix query string being omitted in query stats log. <a href=https://github.com/cortexproject/cortex/pull/5655>#5655</a></li></ul><h2 id=1153-2023-06-22>1.15.3 2023-06-22</h2><ul><li>[BUGFIX] Distributor: Fix potential data corruption in cases of timeout between distributors and ingesters. <a href=https://github.com/cortexproject/cortex/pull/5422>#5422</a></li></ul><h2 id=1152-2023-05-09>1.15.2 2023-05-09</h2><ul><li>[ENHANCEMENT] Update Go version to 1.20.4. <a href=https://github.com/cortexproject/cortex/pull/5299>#5299</a></li></ul><h2 id=1151-2023-04-26>1.15.1 2023-04-26</h2><ul><li>[CHANGE] Alertmanager: Validating new fields on the PagerDuty AM config. <a href=https://github.com/cortexproject/cortex/pull/5290>#5290</a></li><li>[BUGFIX] Querier: Convert gRPC <code>ResourceExhausted</code> status code from store gateway to 422 limit error. <a href=https://github.com/cortexproject/cortex/pull/5286>#5286</a></li></ul><h2 id=1150-2023-04-19>1.15.0 2023-04-19</h2><ul><li>[CHANGE] Storage: Make Max exemplars config per tenant instead of global configuration. <a href=https://github.com/cortexproject/cortex/pull/5080>#5080</a> <a href=https://github.com/cortexproject/cortex/pull/5122>#5122</a></li><li>[CHANGE] Alertmanager: Local file disclosure vulnerability in OpsGenie configuration has been fixed. <a href=https://github.com/cortexproject/cortex/pull/5045>#5045</a></li><li>[CHANGE] Rename oltp_endpoint to otlp_endpoint to match opentelemetry spec and lib name. <a href=https://github.com/cortexproject/cortex/pull/5068>#5068</a></li><li>[CHANGE] Distributor/Ingester: Log warn level on push requests when they have status code 4xx. Do not log if status is 429. <a href=https://github.com/cortexproject/cortex/pull/5103>#5103</a></li><li>[CHANGE] Tracing: Use the default OTEL trace sampler when <code>-tracing.otel.exporter-type</code> is set to <code>awsxray</code>. <a href=https://github.com/cortexproject/cortex/pull/5141>#5141</a></li><li>[CHANGE] Ingester partial error log line to debug level. <a href=https://github.com/cortexproject/cortex/pull/5192>#5192</a></li><li>[CHANGE] Change HTTP status code from 503/422 to 499 if a request is canceled. <a href=https://github.com/cortexproject/cortex/pull/5220>#5220</a></li><li>[CHANGE] Store gateways summary metrics have been converted to histograms <code>cortex_bucket_store_series_blocks_queried</code>, <code>cortex_bucket_store_series_data_fetched</code>, <code>cortex_bucket_store_series_data_size_touched_bytes</code>, <code>cortex_bucket_store_series_data_size_fetched_bytes</code>, <code>cortex_bucket_store_series_data_touched</code>, <code>cortex_bucket_store_series_result_series</code> <a href=https://github.com/cortexproject/cortex/pull/5239>#5239</a></li><li>[FEATURE] Querier/Query Frontend: support Prometheus /api/v1/status/buildinfo API. <a href=https://github.com/cortexproject/cortex/pull/4978>#4978</a></li><li>[FEATURE] Ingester: Add active series to all_user_stats page. <a href=https://github.com/cortexproject/cortex/pull/4972>#4972</a></li><li>[FEATURE] Ingester: Added <code>-blocks-storage.tsdb.head-chunks-write-queue-size</code> allowing to configure the size of the in-memory queue used before flushing chunks to the disk . <a href=https://github.com/cortexproject/cortex/pull/5000>#5000</a></li><li>[FEATURE] Query Frontend: Log query params in query frontend even if error happens. <a href=https://github.com/cortexproject/cortex/pull/5005>#5005</a></li><li>[FEATURE] Ingester: Enable snapshotting of In-memory TSDB on disk during shutdown via <code>-blocks-storage.tsdb.memory-snapshot-on-shutdown</code>. <a href=https://github.com/cortexproject/cortex/pull/5011>#5011</a></li><li>[FEATURE] Query Frontend/Scheduler: Add a new counter metric <code>cortex_request_queue_requests_total</code> for total requests going to queue. <a href=https://github.com/cortexproject/cortex/pull/5030>#5030</a></li><li>[FEATURE] Build ARM docker images. <a href=https://github.com/cortexproject/cortex/pull/5041>#5041</a></li><li>[FEATURE] Query-frontend/Querier: Create spans to measure time to merge promql responses. <a href=https://github.com/cortexproject/cortex/pull/5041>#5041</a></li><li>[FEATURE] Querier/Ruler: Support the new thanos promql engine. This is an experimental feature and might change in the future. <a href=https://github.com/cortexproject/cortex/pull/5093>#5093</a></li><li>[FEATURE] Added zstd as an option for grpc compression <a href=https://github.com/cortexproject/cortex/pull/5092>#5092</a></li><li>[FEATURE] Ring: Add new kv store option <code>dynamodb</code>. <a href=https://github.com/cortexproject/cortex/pull/5026>#5026</a></li><li>[FEATURE] Cache: Support redis as backend for caching bucket and index cache. <a href=https://github.com/cortexproject/cortex/pull/5057>#5057</a></li><li>[FEATURE] Querier/Store-Gateway: Added <code>-blocks-storage.bucket-store.ignore-blocks-within</code> allowing to filter out the recently created blocks from being synced by queriers and store-gateways. <a href=https://github.com/cortexproject/cortex/pull/5166>#5166</a></li><li>[FEATURE] AlertManager/Ruler: Added support for <code>keep_firing_for</code> on alerting rulers.</li><li>[FEATURE] Alertmanager: Add support for time_intervals. <a href=https://github.com/cortexproject/cortex/pull/5102>#5102</a></li><li>[FEATURE] Added <code>snappy-block</code> as an option for grpc compression <a href=https://github.com/cortexproject/cortex/pull/5215>#5215</a></li><li>[FEATURE] Enable experimental out-of-order samples support. Added 2 new configs <code>ingester.out_of_order_time_window</code> and <code>blocks-storage.tsdb.out_of_order_cap_max</code>. <a href=https://github.com/cortexproject/cortex/pull/4964>#4964</a></li><li>[ENHANCEMENT] Querier: limit series query to only ingesters if <code>start</code> param is not specified. <a href=https://github.com/cortexproject/cortex/pull/4976>#4976</a></li><li>[ENHANCEMENT] Query-frontend/scheduler: add a new limit <code>frontend.max-outstanding-requests-per-tenant</code> for configuring queue size per tenant. Started deprecating two flags <code>-query-scheduler.max-outstanding-requests-per-tenant</code> and <code>-querier.max-outstanding-requests-per-tenant</code>, and change their value default to 0. Now if both the old flag and new flag are specified, the old flag&rsquo;s queue size will be picked. <a href=https://github.com/cortexproject/cortex/pull/4991>#4991</a></li><li>[ENHANCEMENT] Query-tee: Add <code>/api/v1/query_exemplars</code> API endpoint support. <a href=https://github.com/cortexproject/cortex/pull/5010>#5010</a></li><li>[ENHANCEMENT] Let blocks_cleaner delete blocks concurrently(default 16 goroutines). <a href=https://github.com/cortexproject/cortex/pull/5028>#5028</a></li><li>[ENHANCEMENT] Query Frontend/Query Scheduler: Increase upper bound to 60s for queue duration histogram metric. <a href=https://github.com/cortexproject/cortex/pull/5029>#5029</a></li><li>[ENHANCEMENT] Query Frontend: Log Vertical sharding information when <code>query_stats_enabled</code> is enabled. <a href=https://github.com/cortexproject/cortex/pull/5037>#5037</a></li><li>[ENHANCEMENT] Ingester: The metadata APIs should honour <code>querier.query-ingesters-within</code> when <code>querier.query-store-for-labels-enabled</code> is true. <a href=https://github.com/cortexproject/cortex/pull/5027>#5027</a></li><li>[ENHANCEMENT] Query Frontend: Skip instant query roundtripper if sharding is not applicable. <a href=https://github.com/cortexproject/cortex/pull/5062>#5062</a></li><li>[ENHANCEMENT] Push reduce one hash operation of Labels. <a href=https://github.com/cortexproject/cortex/pull/4945>#4945</a> <a href=https://github.com/cortexproject/cortex/pull/5114>#5114</a></li><li>[ENHANCEMENT] Alertmanager: Added <code>-alertmanager.enabled-tenants</code> and <code>-alertmanager.disabled-tenants</code> to explicitly enable or disable alertmanager for specific tenants. <a href=https://github.com/cortexproject/cortex/pull/5116>#5116</a></li><li>[ENHANCEMENT] Upgraded Docker base images to <code>alpine:3.17</code>. <a href=https://github.com/cortexproject/cortex/pull/5132>#5132</a></li><li>[ENHANCEMENT] Add retry logic to S3 bucket client. <a href=https://github.com/cortexproject/cortex/pull/5135>#5135</a></li><li>[ENHANCEMENT] Update Go version to 1.20.1. <a href=https://github.com/cortexproject/cortex/pull/5159>#5159</a></li><li>[ENHANCEMENT] Distributor: Reuse byte slices when serializing requests from distributors to ingesters. <a href=https://github.com/cortexproject/cortex/pull/5193>#5193</a></li><li>[ENHANCEMENT] Query Frontend: Add number of chunks and samples fetched in query stats. <a href=https://github.com/cortexproject/cortex/pull/5198>#5198</a></li><li>[ENHANCEMENT] Implement grpc.Compressor.DecompressedSize for snappy to optimize memory allocations. <a href=https://github.com/cortexproject/cortex/pull/5213>#5213</a></li><li>[ENHANCEMENT] Querier: Batch Iterator optimization to prevent transversing it multiple times query ranges steps does not overlap. <a href=https://github.com/cortexproject/cortex/pull/5237>#5237</a></li><li>[BUGFIX] Updated <code>golang.org/x/net</code> dependency to fix CVE-2022-27664. <a href=https://github.com/cortexproject/cortex/pull/5008>#5008</a></li><li>[BUGFIX] Fix panic when otel and xray tracing is enabled. <a href=https://github.com/cortexproject/cortex/pull/5044>#5044</a></li><li>[BUGFIX] Fixed no compact block got grouped in shuffle sharding grouper. <a href=https://github.com/cortexproject/cortex/pull/5055>#5055</a></li><li>[BUGFIX] Fixed ingesters with less tokens stuck in LEAVING. <a href=https://github.com/cortexproject/cortex/pull/5061>#5061</a></li><li>[BUGFIX] Tracing: Fix missing object storage span instrumentation. <a href=https://github.com/cortexproject/cortex/pull/5074>#5074</a></li><li>[BUGFIX] Ingester: Fix Ingesters returning empty response for metadata APIs. <a href=https://github.com/cortexproject/cortex/pull/5081>#5081</a></li><li>[BUGFIX] Ingester: Fix panic when querying metadata from blocks that are being deleted. <a href=https://github.com/cortexproject/cortex/pull/5119>#5119</a></li><li>[BUGFIX] Ring: Fix case when dynamodb kv reaches the limit of 25 actions per batch call. <a href=https://github.com/cortexproject/cortex/pull/5136>#5136</a></li><li>[BUGFIX] Query-frontend: Fix shardable instant queries do not produce sorted results for <code>sort</code>, <code>sort_desc</code>, <code>topk</code>, <code>bottomk</code> functions. <a href=https://github.com/cortexproject/cortex/pull/5148>#5148</a>, <a href=https://github.com/cortexproject/cortex/pull/5170>#5170</a></li><li>[BUGFIX] Querier: Fix <code>/api/v1/series</code> returning 5XX instead of 4XX when limits are hit. <a href=https://github.com/cortexproject/cortex/pull/5169>#5169</a></li><li>[BUGFIX] Compactor: Fix issue that shuffle sharding planner return error if block is under visit by other compactor. <a href=https://github.com/cortexproject/cortex/pull/5188>#5188</a></li><li>[BUGFIX] Fix S3 BucketWithRetries upload empty content issue <a href=https://github.com/cortexproject/cortex/pull/5217>#5217</a></li><li>[BUGFIX] Query Frontend: Disable <code>absent</code>, <code>absent_over_time</code> and <code>scalar</code> for vertical sharding. <a href=https://github.com/cortexproject/cortex/pull/5221>#5221</a></li><li>[BUGFIX] Catch context error in the s3 bucket client. <a href=https://github.com/cortexproject/cortex/pull/5240>#5240</a></li><li>[BUGFIX] Fix query frontend remote read empty body. <a href=https://github.com/cortexproject/cortex/pull/5257>#5257</a></li><li>[BUGFIX] Fix query frontend incorrect error response format at <code>SplitByQuery</code> middleware. <a href=https://github.com/cortexproject/cortex/pull/5260>#5260</a></li></ul><h2 id=1140-2022-12-02>1.14.0 2022-12-02</h2><p><strong>This release removes support for chunks storage. See below for more.</strong></p><ul><li>[CHANGE] Remove support for chunks storage entirely. If you are using chunks storage on a previous version, you must <a href=https://github.com/cortexproject/cortex/blob/v1.13.1/docs/blocks-storage/migrate-from-chunks-to-blocks.md>migrate your data</a> on version 1.13.1 or earlier. Before upgrading to this release, you should also remove any deprecated chunks-related configuration, as this release will no longer accept that. The following flags are gone:<ul><li><code>-dynamodb.*</code></li><li><code>-metrics.*</code></li><li><code>-s3.*</code></li><li><code>-azure.*</code></li><li><code>-bigtable.*</code></li><li><code>-gcs.*</code></li><li><code>-cassandra.*</code></li><li><code>-boltdb.*</code></li><li><code>-local.*</code></li><li>some <code>-ingester</code> flags:<ul><li><code>-ingester.wal-enabled</code></li><li><code>-ingester.checkpoint-enabled</code></li><li><code>-ingester.recover-from-wal</code></li><li><code>-ingester.wal-dir</code></li><li><code>-ingester.checkpoint-duration</code></li><li><code>-ingester.flush-on-shutdown-with-wal-enabled</code></li><li><code>-ingester.max-transfer-retries</code></li><li><code>-ingester.max-samples-per-query</code></li><li><code>-ingester.min-chunk-length</code></li><li><code>-ingester.flush-period</code></li><li><code>-ingester.retain-period</code></li><li><code>-ingester.max-chunk-idle</code></li><li><code>-ingester.max-stale-chunk-idle</code></li><li><code>-ingester.flush-op-timeout</code></li><li><code>-ingester.max-chunk-age</code></li><li><code>-ingester.chunk-age-jitter</code></li><li><code>-ingester.concurrent-flushes</code></li><li><code>-ingester.spread-flushes</code></li><li><code>-ingester.chunk-encoding</code></li><li><code>-store.*</code> except <code>-store.engine</code> and <code>-store.max-query-length</code></li><li><code>-store.query-chunk-limit</code> was deprecated and replaced by <code>-querier.max-fetched-chunks-per-query</code></li></ul></li><li><code>-deletes.*</code></li><li><code>-grpc-store.*</code></li><li><code>-flusher.wal-dir</code>, <code>-flusher.concurrent-flushes</code>, <code>-flusher.flush-op-timeout</code></li></ul></li><li>[CHANGE] Remove support for alertmanager and ruler legacy store configuration. Before upgrading, you need to convert your configuration to use the <code>alertmanager-storage</code> and <code>ruler-storage</code> configuration on the version that you&rsquo;re already running, then upgrade.</li><li>[CHANGE] Disables TSDB isolation. <a href=https://github.com/cortexproject/cortex/pull/4825>#4825</a></li><li>[CHANGE] Drops support Prometheus 1.x rule format on configdb. <a href=https://github.com/cortexproject/cortex/pull/4826>#4826</a></li><li>[CHANGE] Removes <code>-ingester.stream-chunks-when-using-blocks</code> experimental flag and stream chunks by default when <code>querier.ingester-streaming</code> is enabled. <a href=https://github.com/cortexproject/cortex/pull/4864>#4864</a></li><li>[CHANGE] Compactor: Added <code>cortex_compactor_runs_interrupted_total</code> to separate compaction interruptions from failures</li><li>[CHANGE] Enable PromQL <code>@</code> modifier, negative offset always. <a href=https://github.com/cortexproject/cortex/pull/4927>#4927</a></li><li>[CHANGE] Store-gateway: Add user label to <code>cortex_bucket_store_blocks_loaded</code> metric. <a href=https://github.com/cortexproject/cortex/pull/4918>#4918</a></li><li>[CHANGE] AlertManager: include <code>status</code> label in <code>cortex_alertmanager_alerts_received_total</code>. <a href=https://github.com/cortexproject/cortex/pull/4907>#4907</a></li><li>[FEATURE] Compactor: Added <code>-compactor.block-files-concurrency</code> allowing to configure number of go routines for download/upload block files during compaction. <a href=https://github.com/cortexproject/cortex/pull/4784>#4784</a></li><li>[FEATURE] Compactor: Added <code>-compactor.blocks-fetch-concurrency</code> allowing to configure number of go routines for blocks during compaction. <a href=https://github.com/cortexproject/cortex/pull/4787>#4787</a></li><li>[FEATURE] Compactor: Added configurations for Azure MSI in blocks-storage, ruler-storage and alertmanager-storage. <a href=https://github.com/cortexproject/cortex/pull/4818>#4818</a></li><li>[FEATURE] Ruler: Add support to pass custom implementations of queryable and pusher. <a href=https://github.com/cortexproject/cortex/pull/4782>#4782</a></li><li>[FEATURE] Create OpenTelemetry Bridge for Tracing. Now cortex can send traces to multiple destinations using OTEL Collectors. <a href=https://github.com/cortexproject/cortex/pull/4834>#4834</a></li><li>[FEATURE] Added <code>-api.http-request-headers-to-log</code> allowing for the addition of HTTP Headers to logs <a href=https://github.com/cortexproject/cortex/pull/4803>#4803</a></li><li>[FEATURE] Distributor: Added a new limit <code>-validation.max-labels-size-bytes</code> allowing to limit the combined size of labels for each timeseries. <a href=https://github.com/cortexproject/cortex/pull/4848>#4848</a></li><li>[FEATURE] Storage/Bucket: Added <code>-*.s3.bucket-lookup-type</code> allowing to configure the s3 bucket lookup type. <a href=https://github.com/cortexproject/cortex/pull/4794>#4794</a></li><li>[FEATURE] QueryFrontend: Implement experimental vertical sharding at query frontend for range/instant queries. <a href=https://github.com/cortexproject/cortex/pull/4863>#4863</a></li><li>[FEATURE] QueryFrontend: Support vertical sharding for subqueries. <a href=https://github.com/cortexproject/cortex/pull/4955>#4955</a></li><li>[FEATURE] Querier: Added a new limit <code>-querier.max-fetched-data-bytes-per-query</code> allowing to limit the maximum size of all data in bytes that a query can fetch from each ingester and storage. <a href=https://github.com/cortexproject/cortex/pull/4854>#4854</a></li><li>[FEATURE] Added 2 flags <code>-alertmanager.alertmanager-client.grpc-compression</code> and <code>-querier.store-gateway-client.grpc-compression</code> to configure compression methods for grpc clients. <a href=https://github.com/cortexproject/cortex/pull/4889>#4889</a></li><li>[ENHANCEMENT] AlertManager: Retrying AlertManager Get Requests (Get Alertmanager status, Get Alertmanager Receivers) on next replica on error <a href=https://github.com/cortexproject/cortex/pull/4840>#4840</a></li><li>[ENHANCEMENT] Querier/Ruler: Retry store-gateway in case of unexpected failure, instead of failing the query. <a href=https://github.com/cortexproject/cortex/pull/4532>#4532</a> <a href=https://github.com/cortexproject/cortex/pull/4839>#4839</a></li><li>[ENHANCEMENT] Ring: DoBatch prioritize 4xx errors when failing. <a href=https://github.com/cortexproject/cortex/pull/4783>#4783</a></li><li>[ENHANCEMENT] Cortex now built with Go 1.18. <a href=https://github.com/cortexproject/cortex/pull/4829>#4829</a></li><li>[ENHANCEMENT] Ingester: Prevent ingesters to become unhealthy during wall replay. <a href=https://github.com/cortexproject/cortex/pull/4847>#4847</a></li><li>[ENHANCEMENT] Compactor: Introduced visit marker file for blocks so blocks are under compaction will not be picked up by another compactor. <a href=https://github.com/cortexproject/cortex/pull/4805>#4805</a></li><li>[ENHANCEMENT] Distributor: Add label name to labelValueTooLongError. <a href=https://github.com/cortexproject/cortex/pull/4855>#4855</a></li><li>[ENHANCEMENT] Enhance traces with hostname information. <a href=https://github.com/cortexproject/cortex/pull/4898>#4898</a></li><li>[ENHANCEMENT] Improve the documentation around limits. <a href=https://github.com/cortexproject/cortex/pull/4905>#4905</a></li><li>[ENHANCEMENT] Distributor: cache user overrides to reduce lock contention. <a href=https://github.com/cortexproject/cortex/pull/4904>#4904</a></li><li>[BUGFIX] Storage/Bucket: Enable AWS SDK for go authentication for s3 to fix IMDSv1 authentication. <a href=https://github.com/cortexproject/cortex/pull/4897>#4897</a></li><li>[BUGFIX] Memberlist: Add join with no retrying when starting service. <a href=https://github.com/cortexproject/cortex/pull/4804>#4804</a></li><li>[BUGFIX] Ruler: Fix /ruler/rule_groups returns YAML with extra fields. <a href=https://github.com/cortexproject/cortex/pull/4767>#4767</a></li><li>[BUGFIX] Respecting <code>-tracing.otel.sample-ratio</code> configuration when enabling OpenTelemetry tracing with X-ray. <a href=https://github.com/cortexproject/cortex/pull/4862>#4862</a></li><li>[BUGFIX] QueryFrontend: fixed query_range requests when query has <code>start</code> equals to <code>end</code>. <a href=https://github.com/cortexproject/cortex/pull/4877>#4877</a></li><li>[BUGFIX] AlertManager: fixed issue introduced by <a href=https://github.com/cortexproject/cortex/pull/4495>#4495</a> where templates files were being deleted when using alertmanager local store. <a href=https://github.com/cortexproject/cortex/pull/4890>#4890</a></li><li>[BUGFIX] Ingester: fixed incorrect logging at the start of ingester block shipping logic. <a href=https://github.com/cortexproject/cortex/pull/4934>#4934</a></li><li>[BUGFIX] Storage/Bucket: fixed global mark missing on deletion. <a href=https://github.com/cortexproject/cortex/pull/4949>#4949</a></li><li>[BUGFIX] QueryFrontend/Querier: fixed regression added by <a href=https://github.com/cortexproject/cortex/pull/4863>#4863</a> where we stopped compressing the response between querier and query frontend. <a href=https://github.com/cortexproject/cortex/pull/4960>#4960</a></li><li>[BUGFIX] QueryFrontend/Querier: fixed fix response error to be ungzipped when status code is not 2xx. <a href=https://github.com/cortexproject/cortex/pull/4975>#4975</a></li></ul><h3 id=known-issues>Known issues</h3><ul><li>Configsdb: Ruler configs doesn&rsquo;t work. Remove all configs from postgres database that have format Prometheus 1.x rule format before upgrading to v1.14.0 (see <a href=https://github.com/cortexproject/cortex/issues/5387>5387</a>)</li></ul><h2 id=1130-2022-07-14>1.13.0 2022-07-14</h2><ul><li>[CHANGE] Changed default for <code>-ingester.min-ready-duration</code> from 1 minute to 15 seconds. <a href=https://github.com/cortexproject/cortex/pull/4539>#4539</a></li><li>[CHANGE] query-frontend: Do not print anything in the logs of <code>query-frontend</code> if a in-progress query has been canceled (context canceled) to avoid spam. <a href=https://github.com/cortexproject/cortex/pull/4562>#4562</a></li><li>[CHANGE] Compactor block deletion mark migration, needed when upgrading from v1.7, is now disabled by default. <a href=https://github.com/cortexproject/cortex/pull/4597>#4597</a></li><li>[CHANGE] The <code>status_code</code> label on gRPC client metrics has changed from &lsquo;200&rsquo; and &lsquo;500&rsquo; to &lsquo;2xx&rsquo;, &lsquo;5xx&rsquo;, &lsquo;4xx&rsquo;, &lsquo;cancel&rsquo; or &rsquo;error&rsquo;. <a href=https://github.com/cortexproject/cortex/pull/4601>#4601</a></li><li>[CHANGE] Memberlist: changed probe interval from <code>1s</code> to <code>5s</code> and probe timeout from <code>500ms</code> to <code>2s</code>. <a href=https://github.com/cortexproject/cortex/pull/4601>#4601</a></li><li>[CHANGE] Fix incorrectly named <code>cortex_cache_fetched_keys</code> and <code>cortex_cache_hits</code> metrics. Renamed to <code>cortex_cache_fetched_keys_total</code> and <code>cortex_cache_hits_total</code> respectively. <a href=https://github.com/cortexproject/cortex/pull/4686>#4686</a></li><li>[CHANGE] Enable Thanos series limiter in store-gateway. <a href=https://github.com/cortexproject/cortex/pull/4702>#4702</a></li><li>[CHANGE] Distributor: Apply <code>max_fetched_series_per_query</code> limit for <code>/series</code> API. <a href=https://github.com/cortexproject/cortex/pull/4683>#4683</a></li><li>[CHANGE] Re-enable the <code>proxy_url</code> option for receiver configuration. <a href=https://github.com/cortexproject/cortex/pull/4741>#4741</a></li><li>[FEATURE] Ruler: Add <code>external_labels</code> option to tag all alerts with a given set of labels. <a href=https://github.com/cortexproject/cortex/pull/4499>#4499</a></li><li>[FEATURE] Compactor: Add <code>-compactor.skip-blocks-with-out-of-order-chunks-enabled</code> configuration to mark blocks containing index with out-of-order chunks for no compact instead of halting the compaction. <a href=https://github.com/cortexproject/cortex/pull/4707>#4707</a></li><li>[FEATURE] Querier/Query-Frontend: Add <code>-querier.per-step-stats-enabled</code> and <code>-frontend.cache-queryable-samples-stats</code> configurations to enable query sample statistics. <a href=https://github.com/cortexproject/cortex/pull/4708>#4708</a></li><li>[FEATURE] Add shuffle sharding for the compactor <a href=https://github.com/cortexproject/cortex/pull/4433>#4433</a></li><li>[FEATURE] Querier: Use streaming for ingester metadata APIs. <a href=https://github.com/cortexproject/cortex/pull/4725>#4725</a></li><li>[ENHANCEMENT] Update Go version to 1.17.8. <a href=https://github.com/cortexproject/cortex/pull/4602>#4602</a> <a href=https://github.com/cortexproject/cortex/pull/4604>#4604</a> <a href=https://github.com/cortexproject/cortex/pull/4658>#4658</a></li><li>[ENHANCEMENT] Keep track of discarded samples due to bad relabel configuration in <code>cortex_discarded_samples_total</code>. <a href=https://github.com/cortexproject/cortex/pull/4503>#4503</a></li><li>[ENHANCEMENT] Ruler: Add <code>-ruler.disable-rule-group-label</code> to disable the <code>rule_group</code> label on exported metrics. <a href=https://github.com/cortexproject/cortex/pull/4571>#4571</a></li><li>[ENHANCEMENT] Query federation: improve performance in MergeQueryable by memoizing labels. <a href=https://github.com/cortexproject/cortex/pull/4502>#4502</a></li><li>[ENHANCEMENT] Added new ring related config <code>-ingester.readiness-check-ring-health</code> when enabled the readiness probe will succeed only after all instances are ACTIVE and healthy in the ring, this is enabled by default. <a href=https://github.com/cortexproject/cortex/pull/4539>#4539</a></li><li>[ENHANCEMENT] Added new ring related config <code>-distributor.excluded-zones</code> when set this will exclude the comma-separated zones from the ring, default is &ldquo;&rdquo;. <a href=https://github.com/cortexproject/cortex/pull/4539>#4539</a></li><li>[ENHANCEMENT] Upgraded Docker base images to <code>alpine:3.14</code>. <a href=https://github.com/cortexproject/cortex/pull/4514>#4514</a></li><li>[ENHANCEMENT] Updated Prometheus to latest. Includes changes from prometheus<a href=https://github.com/cortexproject/cortex/pull/9239>#9239</a>, adding 15 new functions. Multiple TSDB bugfixes prometheus<a href=https://github.com/cortexproject/cortex/pull/9438>#9438</a> & prometheus<a href=https://github.com/cortexproject/cortex/pull/9381>#9381</a>. <a href=https://github.com/cortexproject/cortex/pull/4524>#4524</a></li><li>[ENHANCEMENT] Query Frontend: Add setting <code>-frontend.forward-headers-list</code> in frontend to configure the set of headers from the requests to be forwarded to downstream requests. <a href=https://github.com/cortexproject/cortex/pull/4486>#4486</a></li><li>[ENHANCEMENT] Blocks storage: Add <code>-blocks-storage.azure.http.*</code>, <code>-alertmanager-storage.azure.http.*</code>, and <code>-ruler-storage.azure.http.*</code> to configure the Azure storage client. <a href=https://github.com/cortexproject/cortex/pull/4581>#4581</a></li><li>[ENHANCEMENT] Optimise memberlist receive path when used as a backing store for rings with a large number of members. <a href=https://github.com/cortexproject/cortex/pull/4601>#4601</a></li><li>[ENHANCEMENT] Add length and limit to labelNameTooLongError and labelValueTooLongError <a href=https://github.com/cortexproject/cortex/pull/4595>#4595</a></li><li>[ENHANCEMENT] Add jitter to rejoinInterval. <a href=https://github.com/cortexproject/cortex/pull/4747>#4747</a></li><li>[ENHANCEMENT] Compactor: uploading blocks no compaction marks to the global location and introduce a new metric <a href=https://github.com/cortexproject/cortex/pull/4729>#4729</a><ul><li><code>cortex_bucket_blocks_marked_for_no_compaction_count</code>: Total number of blocks marked for no compaction in the bucket.</li></ul></li><li>[ENHANCEMENT] Querier: Reduce the number of series that are kept in memory while streaming from ingesters. <a href=https://github.com/cortexproject/cortex/pull/4745>#4745</a></li><li>[BUGFIX] AlertManager: remove stale template files. <a href=https://github.com/cortexproject/cortex/pull/4495>#4495</a></li><li>[BUGFIX] Distributor: fix bug in query-exemplar where some results would get dropped. <a href=https://github.com/cortexproject/cortex/pull/4583>#4583</a></li><li>[BUGFIX] Update Thanos dependency: compactor tracing support, azure blocks storage memory fix. <a href=https://github.com/cortexproject/cortex/pull/4585>#4585</a></li><li>[BUGFIX] Set appropriate <code>Content-Type</code> header for /services endpoint, which previously hard-coded <code>text/plain</code>. <a href=https://github.com/cortexproject/cortex/pull/4596>#4596</a></li><li>[BUGFIX] Querier: Disable query scheduler SRV DNS lookup, which removes noisy log messages about &ldquo;failed DNS SRV record lookup&rdquo;. <a href=https://github.com/cortexproject/cortex/pull/4601>#4601</a></li><li>[BUGFIX] Memberlist: fixed corrupted packets when sending compound messages with more than 255 messages or messages bigger than 64KB. <a href=https://github.com/cortexproject/cortex/pull/4601>#4601</a></li><li>[BUGFIX] Query Frontend: If &lsquo;LogQueriesLongerThan&rsquo; is set to &lt; 0, log all queries as described in the docs. <a href=https://github.com/cortexproject/cortex/pull/4633>#4633</a></li><li>[BUGFIX] Distributor: update defaultReplicationStrategy to not fail with extend-write when a single instance is unhealthy. <a href=https://github.com/cortexproject/cortex/pull/4636>#4636</a></li><li>[BUGFIX] Distributor: Fix race condition on <code>/series</code> introduced by <a href=https://github.com/cortexproject/cortex/pull/4683>#4683</a>. <a href=https://github.com/cortexproject/cortex/pull/4716>#4716</a></li><li>[BUGFIX] Ruler: Fixed leaking notifiers after users are removed <a href=https://github.com/cortexproject/cortex/pull/4718>#4718</a></li><li>[BUGFIX] Distributor: Fix a memory leak in distributor due to the cluster label. <a href=https://github.com/cortexproject/cortex/pull/4739>#4739</a></li><li>[BUGFIX] Memberlist: Avoid clock skew by limiting the timestamp accepted on gossip. <a href=https://github.com/cortexproject/cortex/pull/4750>#4750</a></li><li>[BUGFIX] Compactor: skip compaction if there is only 1 block available for shuffle-sharding compactor. <a href=https://github.com/cortexproject/cortex/pull/4756>#4756</a></li><li>[BUGFIX] Compactor: Fixes <a href=https://github.com/cortexproject/cortex/pull/4770>#4770</a> - an edge case in compactor with shulffle sharding where compaction stops when a tenant stops ingesting samples. <a href=https://github.com/cortexproject/cortex/pull/4771>#4771</a></li><li>[BUGFIX] Compactor: fix cortex_compactor_remaining_planned_compactions not set after plan generation for shuffle sharding compactor. <a href=https://github.com/cortexproject/cortex/pull/4772>#4772</a></li></ul><h2 id=1110-2021-11-25>1.11.0 2021-11-25</h2><ul><li>[CHANGE] Memberlist: Expose default configuration values to the command line options. Note that setting these explicitly to zero will no longer cause the default to be used. If the default is desired, then do set the option. The following are affected: <a href=https://github.com/cortexproject/cortex/pull/4276>#4276</a><ul><li><code>-memberlist.stream-timeout</code></li><li><code>-memberlist.retransmit-factor</code></li><li><code>-memberlist.pull-push-interval</code></li><li><code>-memberlist.gossip-interval</code></li><li><code>-memberlist.gossip-nodes</code></li><li><code>-memberlist.gossip-to-dead-nodes-time</code></li><li><code>-memberlist.dead-node-reclaim-time</code></li></ul></li><li>[CHANGE] <code>-querier.max-fetched-chunks-per-query</code> previously applied to chunks from ingesters and store separately; now the two combined should not exceed the limit. <a href=https://github.com/cortexproject/cortex/pull/4260>#4260</a></li><li>[CHANGE] Memberlist: the metric <code>memberlist_kv_store_value_bytes</code> has been removed due to values no longer being stored in-memory as encoded bytes. <a href=https://github.com/cortexproject/cortex/pull/4345>#4345</a></li><li>[CHANGE] Some files and directories created by Cortex components on local disk now have stricter permissions, and are only readable by owner, but not group or others. <a href=https://github.com/cortexproject/cortex/pull/4394>#4394</a></li><li>[CHANGE] The metric <code>cortex_deprecated_flags_inuse_total</code> has been renamed to <code>deprecated_flags_inuse_total</code> as part of using grafana/dskit functionality. <a href=https://github.com/cortexproject/cortex/pull/4443>#4443</a></li><li>[FEATURE] Ruler: Add new <code>-ruler.query-stats-enabled</code> which when enabled will report the <code>cortex_ruler_query_seconds_total</code> as a per-user metric that tracks the sum of the wall time of executing queries in the ruler in seconds. <a href=https://github.com/cortexproject/cortex/pull/4317>#4317</a></li><li>[FEATURE] Query Frontend: Add <code>cortex_query_fetched_series_total</code> and <code>cortex_query_fetched_chunks_bytes_total</code> per-user counters to expose the number of series and bytes fetched as part of queries. These metrics can be enabled with the <code>-frontend.query-stats-enabled</code> flag (or its respective YAML config option <code>query_stats_enabled</code>). <a href=https://github.com/cortexproject/cortex/pull/4343>#4343</a></li><li>[FEATURE] AlertManager: Add support for SNS Receiver. <a href=https://github.com/cortexproject/cortex/pull/4382>#4382</a></li><li>[FEATURE] Distributor: Add label <code>status</code> to metric <code>cortex_distributor_ingester_append_failures_total</code> <a href=https://github.com/cortexproject/cortex/pull/4442>#4442</a></li><li>[FEATURE] Queries: Added <code>present_over_time</code> PromQL function, also some TSDB optimisations. <a href=https://github.com/cortexproject/cortex/pull/4505>#4505</a></li><li>[ENHANCEMENT] Add timeout for waiting on compactor to become ACTIVE in the ring. <a href=https://github.com/cortexproject/cortex/pull/4262>#4262</a></li><li>[ENHANCEMENT] Reduce memory used by streaming queries, particularly in ruler. <a href=https://github.com/cortexproject/cortex/pull/4341>#4341</a></li><li>[ENHANCEMENT] Ring: allow experimental configuration of disabling of heartbeat timeouts by setting the relevant configuration value to zero. Applies to the following: <a href=https://github.com/cortexproject/cortex/pull/4342>#4342</a><ul><li><code>-distributor.ring.heartbeat-timeout</code></li><li><code>-ring.heartbeat-timeout</code></li><li><code>-ruler.ring.heartbeat-timeout</code></li><li><code>-alertmanager.sharding-ring.heartbeat-timeout</code></li><li><code>-compactor.ring.heartbeat-timeout</code></li><li><code>-store-gateway.sharding-ring.heartbeat-timeout</code></li></ul></li><li>[ENHANCEMENT] Ring: allow heartbeats to be explicitly disabled by setting the interval to zero. This is considered experimental. This applies to the following configuration options: <a href=https://github.com/cortexproject/cortex/pull/4344>#4344</a><ul><li><code>-distributor.ring.heartbeat-period</code></li><li><code>-ingester.heartbeat-period</code></li><li><code>-ruler.ring.heartbeat-period</code></li><li><code>-alertmanager.sharding-ring.heartbeat-period</code></li><li><code>-compactor.ring.heartbeat-period</code></li><li><code>-store-gateway.sharding-ring.heartbeat-period</code></li></ul></li><li>[ENHANCEMENT] Memberlist: optimized receive path for processing ring state updates, to help reduce CPU utilization in large clusters. <a href=https://github.com/cortexproject/cortex/pull/4345>#4345</a></li><li>[ENHANCEMENT] Memberlist: expose configuration of memberlist packet compression via <code>-memberlist.compression=enabled</code>. <a href=https://github.com/cortexproject/cortex/pull/4346>#4346</a></li><li>[ENHANCEMENT] Update Go version to 1.16.6. <a href=https://github.com/cortexproject/cortex/pull/4362>#4362</a></li><li>[ENHANCEMENT] Updated Prometheus to include changes from prometheus/prometheus<a href=https://github.com/cortexproject/cortex/pull/9083>#9083</a>. Now whenever <code>/labels</code> API calls include matchers, blocks store is queried for <code>LabelNames</code> with matchers instead of <code>Series</code> calls which was inefficient. <a href=https://github.com/cortexproject/cortex/pull/4380>#4380</a></li><li>[ENHANCEMENT] Querier: performance improvements in socket and memory handling. <a href=https://github.com/cortexproject/cortex/pull/4429>#4429</a> <a href=https://github.com/cortexproject/cortex/pull/4377>#4377</a></li><li>[ENHANCEMENT] Exemplars are now emitted for all gRPC calls and many operations tracked by histograms. <a href=https://github.com/cortexproject/cortex/pull/4462>#4462</a></li><li>[ENHANCEMENT] New options <code>-server.http-listen-network</code> and <code>-server.grpc-listen-network</code> allow binding as &rsquo;tcp4&rsquo; or &rsquo;tcp6&rsquo;. <a href=https://github.com/cortexproject/cortex/pull/4462>#4462</a></li><li>[ENHANCEMENT] Rulers: Using shuffle sharding subring on GetRules API. <a href=https://github.com/cortexproject/cortex/pull/4466>#4466</a></li><li>[ENHANCEMENT] Support memcached auto-discovery via <code>auto-discovery</code> flag, introduced by thanos in <a href=https://github.com/thanos-io/thanos/pull/4487>https://github.com/thanos-io/thanos/pull/4487</a>. Both AWS and Google Cloud memcached service support auto-discovery, which returns a list of nodes of the memcached cluster. <a href=https://github.com/cortexproject/cortex/pull/4412>#4412</a></li><li>[BUGFIX] Fixes a panic in the query-tee when comparing result. <a href=https://github.com/cortexproject/cortex/pull/4465>#4465</a></li><li>[BUGFIX] Frontend: Fixes @ modifier functions (start/end) when splitting queries by time. <a href=https://github.com/cortexproject/cortex/pull/4464>#4464</a></li><li>[BUGFIX] Compactor: compactor will no longer try to compact blocks that are already marked for deletion. Previously compactor would consider blocks marked for deletion within <code>-compactor.deletion-delay / 2</code> period as eligible for compaction. <a href=https://github.com/cortexproject/cortex/pull/4328>#4328</a></li><li>[BUGFIX] HA Tracker: when cleaning up obsolete elected replicas from KV store, tracker didn&rsquo;t update number of cluster per user correctly. <a href=https://github.com/cortexproject/cortex/pull/4336>#4336</a></li><li>[BUGFIX] Ruler: fixed counting of PromQL evaluation errors as user-errors when updating <code>cortex_ruler_queries_failed_total</code>. <a href=https://github.com/cortexproject/cortex/pull/4335>#4335</a></li><li>[BUGFIX] Ingester: When using block storage, prevent any reads or writes while the ingester is stopping. This will prevent accessing TSDB blocks once they have been already closed. <a href=https://github.com/cortexproject/cortex/pull/4304>#4304</a></li><li>[BUGFIX] Ingester: fixed ingester stuck on start up (LEAVING ring state) when <code>-ingester.heartbeat-period=0</code> and <code>-ingester.unregister-on-shutdown=false</code>. <a href=https://github.com/cortexproject/cortex/pull/4366>#4366</a></li><li>[BUGFIX] Ingester: panic during shutdown while fetching batches from cache. <a href=https://github.com/cortexproject/cortex/pull/4397>#4397</a></li><li>[BUGFIX] Querier: After query-frontend restart, querier may have lower than configured concurrency. <a href=https://github.com/cortexproject/cortex/pull/4417>#4417</a></li><li>[BUGFIX] Memberlist: forward only changes, not entire original message. <a href=https://github.com/cortexproject/cortex/pull/4419>#4419</a></li><li>[BUGFIX] Memberlist: don&rsquo;t accept old tombstones as incoming change, and don&rsquo;t forward such messages to other gossip members. <a href=https://github.com/cortexproject/cortex/pull/4420>#4420</a></li><li>[BUGFIX] Querier: fixed panic when querying exemplars and using <code>-distributor.shard-by-all-labels=false</code>. <a href=https://github.com/cortexproject/cortex/pull/4473>#4473</a></li><li>[BUGFIX] Querier: honor querier minT,maxT if <code>nil</code> SelectHints are passed to Select(). <a href=https://github.com/cortexproject/cortex/pull/4413>#4413</a></li><li>[BUGFIX] Compactor: fixed panic while collecting Prometheus metrics. <a href=https://github.com/cortexproject/cortex/pull/4483>#4483</a></li><li>[BUGFIX] Update go-kit package to fix spurious log messages <a href=https://github.com/cortexproject/cortex/pull/4544>#4544</a></li></ul><h2 id=1100--2021-08-03>1.10.0 / 2021-08-03</h2><ul><li>[CHANGE] Prevent path traversal attack from users able to control the HTTP header <code>X-Scope-OrgID</code>. <a href=https://github.com/cortexproject/cortex/pull/4375>#4375</a> (CVE-2021-36157)<ul><li>Users only have control of the HTTP header when Cortex is not frontend by an auth proxy validating the tenant IDs</li></ul></li><li>[CHANGE] Enable strict JSON unmarshal for <code>pkg/util/validation.Limits</code> struct. The custom <code>UnmarshalJSON()</code> will now fail if the input has unknown fields. <a href=https://github.com/cortexproject/cortex/pull/4298>#4298</a></li><li>[CHANGE] Cortex chunks storage has been deprecated and it&rsquo;s now in maintenance mode: all Cortex users are encouraged to migrate to the blocks storage. No new features will be added to the chunks storage. The default Cortex configuration still runs the chunks engine; please check out the <a href=https://cortexmetrics.io/docs/blocks-storage/>blocks storage doc</a> on how to configure Cortex to run with the blocks storage. <a href=https://github.com/cortexproject/cortex/pull/4268>#4268</a></li><li>[CHANGE] The example Kubernetes manifests (stored at <code>k8s/</code>) have been removed due to a lack of proper support and maintenance. <a href=https://github.com/cortexproject/cortex/pull/4268>#4268</a></li><li>[CHANGE] Querier / ruler: deprecated <code>-store.query-chunk-limit</code> CLI flag (and its respective YAML config option <code>max_chunks_per_query</code>) in favour of <code>-querier.max-fetched-chunks-per-query</code> (and its respective YAML config option <code>max_fetched_chunks_per_query</code>). The new limit specifies the maximum number of chunks that can be fetched in a single query from ingesters and long-term storage: the total number of actual fetched chunks could be 2x the limit, being independently applied when querying ingesters and long-term storage. <a href=https://github.com/cortexproject/cortex/pull/4125>#4125</a></li><li>[CHANGE] Alertmanager: allowed to configure the experimental receivers firewall on a per-tenant basis. The following CLI flags (and their respective YAML config options) have been changed and moved to the limits config section: <a href=https://github.com/cortexproject/cortex/pull/4143>#4143</a><ul><li><code>-alertmanager.receivers-firewall.block.cidr-networks</code> renamed to <code>-alertmanager.receivers-firewall-block-cidr-networks</code></li><li><code>-alertmanager.receivers-firewall.block.private-addresses</code> renamed to <code>-alertmanager.receivers-firewall-block-private-addresses</code></li></ul></li><li>[CHANGE] Change default value of <code>-server.grpc.keepalive.min-time-between-pings</code> from <code>5m</code> to <code>10s</code> and <code>-server.grpc.keepalive.ping-without-stream-allowed</code> to <code>true</code>. <a href=https://github.com/cortexproject/cortex/pull/4168>#4168</a></li><li>[CHANGE] Ingester: Change default value of <code>-ingester.active-series-metrics-enabled</code> to <code>true</code>. This incurs a small increase in memory usage, between 1.2% and 1.6% as measured on ingesters with 1.3M active series. <a href=https://github.com/cortexproject/cortex/pull/4257>#4257</a></li><li>[CHANGE] Dependency: update go-redis from v8.2.3 to v8.9.0. <a href=https://github.com/cortexproject/cortex/pull/4236>#4236</a></li><li>[FEATURE] Querier: Added new <code>-querier.max-fetched-series-per-query</code> flag. When Cortex is running with blocks storage, the max series per query limit is enforced in the querier and applies to unique series received from ingesters and store-gateway (long-term storage). <a href=https://github.com/cortexproject/cortex/pull/4179>#4179</a></li><li>[FEATURE] Querier/Ruler: Added new <code>-querier.max-fetched-chunk-bytes-per-query</code> flag. When Cortex is running with blocks storage, the max chunk bytes limit is enforced in the querier and ruler and limits the size of all aggregated chunks returned from ingesters and storage as bytes for a query. <a href=https://github.com/cortexproject/cortex/pull/4216>#4216</a></li><li>[FEATURE] Alertmanager: support negative matchers, time-based muting - <a href=https://github.com/prometheus/alertmanager/releases/tag/v0.22.0>upstream release notes</a>. <a href=https://github.com/cortexproject/cortex/pull/4237>#4237</a></li><li>[FEATURE] Alertmanager: Added rate-limits to notifiers. Rate limits used by all integrations can be configured using <code>-alertmanager.notification-rate-limit</code>, while per-integration rate limits can be specified via <code>-alertmanager.notification-rate-limit-per-integration</code> parameter. Both shared and per-integration limits can be overwritten using overrides mechanism. These limits are applied on individual (per-tenant) alertmanagers. Rate-limited notifications are failed notifications. It is possible to monitor rate-limited notifications via new <code>cortex_alertmanager_notification_rate_limited_total</code> metric. <a href=https://github.com/cortexproject/cortex/pull/4135>#4135</a> <a href=https://github.com/cortexproject/cortex/pull/4163>#4163</a></li><li>[FEATURE] Alertmanager: Added <code>-alertmanager.max-config-size-bytes</code> limit to control size of configuration files that Cortex users can upload to Alertmanager via API. This limit is configurable per-tenant. <a href=https://github.com/cortexproject/cortex/pull/4201>#4201</a></li><li>[FEATURE] Alertmanager: Added <code>-alertmanager.max-templates-count</code> and <code>-alertmanager.max-template-size-bytes</code> options to control number and size of templates uploaded to Alertmanager via API. These limits are configurable per-tenant. <a href=https://github.com/cortexproject/cortex/pull/4223>#4223</a></li><li>[FEATURE] Added flag <code>-debug.block-profile-rate</code> to enable goroutine blocking events profiling. <a href=https://github.com/cortexproject/cortex/pull/4217>#4217</a></li><li>[FEATURE] Alertmanager: The experimental sharding feature is now considered complete. Detailed information about the configuration options can be found <a href=https://cortexmetrics.io/docs/configuration/configuration-file/#alertmanager_config>here for alertmanager</a> and <a href=https://cortexmetrics.io/docs/configuration/configuration-file/#alertmanager_storage_config>here for the alertmanager storage</a>. To use the feature: <a href=https://github.com/cortexproject/cortex/pull/3925>#3925</a> <a href=https://github.com/cortexproject/cortex/pull/4020>#4020</a> <a href=https://github.com/cortexproject/cortex/pull/4021>#4021</a> <a href=https://github.com/cortexproject/cortex/pull/4031>#4031</a> <a href=https://github.com/cortexproject/cortex/pull/4084>#4084</a> <a href=https://github.com/cortexproject/cortex/pull/4110>#4110</a> <a href=https://github.com/cortexproject/cortex/pull/4126>#4126</a> <a href=https://github.com/cortexproject/cortex/pull/4127>#4127</a> <a href=https://github.com/cortexproject/cortex/pull/4141>#4141</a> <a href=https://github.com/cortexproject/cortex/pull/4146>#4146</a> <a href=https://github.com/cortexproject/cortex/pull/4161>#4161</a> <a href=https://github.com/cortexproject/cortex/pull/4162>#4162</a> <a href=https://github.com/cortexproject/cortex/pull/4222>#4222</a><ul><li>Ensure that a remote storage backend is configured for Alertmanager to store state using <code>-alertmanager-storage.backend</code>, and flags related to the backend. Note that the <code>local</code> and <code>configdb</code> storage backends are not supported.</li><li>Ensure that a ring store is configured using <code>-alertmanager.sharding-ring.store</code>, and set the flags relevant to the chosen store type.</li><li>Enable the feature using <code>-alertmanager.sharding-enabled</code>.</li><li>Note the prior addition of a new configuration option <code>-alertmanager.persist-interval</code>. This sets the interval between persisting the current alertmanager state (notification log and silences) to object storage. See the <a href=https://cortexmetrics.io/docs/configuration/configuration-file/#alertmanager_config>configuration file reference</a> for more information.</li></ul></li><li>[ENHANCEMENT] Alertmanager: Cleanup persisted state objects from remote storage when a tenant configuration is deleted. <a href=https://github.com/cortexproject/cortex/pull/4167>#4167</a></li><li>[ENHANCEMENT] Storage: Added the ability to disable Open Census within GCS client (e.g <code>-gcs.enable-opencensus=false</code>). <a href=https://github.com/cortexproject/cortex/pull/4219>#4219</a></li><li>[ENHANCEMENT] Etcd: Added username and password to etcd config. <a href=https://github.com/cortexproject/cortex/pull/4205>#4205</a></li><li>[ENHANCEMENT] Alertmanager: introduced new metrics to monitor operation when using <code>-alertmanager.sharding-enabled</code>: <a href=https://github.com/cortexproject/cortex/pull/4149>#4149</a><ul><li><code>cortex_alertmanager_state_fetch_replica_state_total</code></li><li><code>cortex_alertmanager_state_fetch_replica_state_failed_total</code></li><li><code>cortex_alertmanager_state_initial_sync_total</code></li><li><code>cortex_alertmanager_state_initial_sync_completed_total</code></li><li><code>cortex_alertmanager_state_initial_sync_duration_seconds</code></li><li><code>cortex_alertmanager_state_persist_total</code></li><li><code>cortex_alertmanager_state_persist_failed_total</code></li></ul></li><li>[ENHANCEMENT] Blocks storage: support ingesting exemplars and querying of exemplars. Enabled by setting new CLI flag <code>-blocks-storage.tsdb.max-exemplars=&lt;n></code> or config option <code>blocks_storage.tsdb.max_exemplars</code> to positive value. <a href=https://github.com/cortexproject/cortex/pull/4124>#4124</a> <a href=https://github.com/cortexproject/cortex/pull/4181>#4181</a></li><li>[ENHANCEMENT] Distributor: Added distributors ring status section in the admin page. <a href=https://github.com/cortexproject/cortex/pull/4151>#4151</a></li><li>[ENHANCEMENT] Added zone-awareness support to alertmanager for use when sharding is enabled. When zone-awareness is enabled, alerts will be replicated across availability zones. <a href=https://github.com/cortexproject/cortex/pull/4204>#4204</a></li><li>[ENHANCEMENT] Added <code>tenant_ids</code> tag to tracing spans <a href=https://github.com/cortexproject/cortex/pull/4186>#4186</a></li><li>[ENHANCEMENT] Ring, query-frontend: Avoid using automatic private IPs (APIPA) when discovering IP address from the interface during the registration of the instance in the ring, or by query-frontend when used with query-scheduler. APIPA still used as last resort with logging indicating usage. <a href=https://github.com/cortexproject/cortex/pull/4032>#4032</a></li><li>[ENHANCEMENT] Memberlist: introduced new metrics to aid troubleshooting tombstone convergence: <a href=https://github.com/cortexproject/cortex/pull/4231>#4231</a><ul><li><code>memberlist_client_kv_store_value_tombstones</code></li><li><code>memberlist_client_kv_store_value_tombstones_removed_total</code></li><li><code>memberlist_client_messages_to_broadcast_dropped_total</code></li></ul></li><li>[ENHANCEMENT] Alertmanager: Added <code>-alertmanager.max-dispatcher-aggregation-groups</code> option to control max number of active dispatcher groups in Alertmanager (per tenant, also overridable). When the limit is reached, Dispatcher produces log message and increases <code>cortex_alertmanager_dispatcher_aggregation_group_limit_reached_total</code> metric. <a href=https://github.com/cortexproject/cortex/pull/4254>#4254</a></li><li>[ENHANCEMENT] Alertmanager: Added <code>-alertmanager.max-alerts-count</code> and <code>-alertmanager.max-alerts-size-bytes</code> to control max number of alerts and total size of alerts that a single user can have in Alertmanager&rsquo;s memory. Adding more alerts will fail with a log message and incrementing <code>cortex_alertmanager_alerts_insert_limited_total</code> metric (per-user). These limits can be overrided by using per-tenant overrides. Current values are tracked in <code>cortex_alertmanager_alerts_limiter_current_alerts</code> and <code>cortex_alertmanager_alerts_limiter_current_alerts_size_bytes</code> metrics. <a href=https://github.com/cortexproject/cortex/pull/4253>#4253</a></li><li>[ENHANCEMENT] Store-gateway: added <code>-store-gateway.sharding-ring.wait-stability-min-duration</code> and <code>-store-gateway.sharding-ring.wait-stability-max-duration</code> support to store-gateway, to wait for ring stability at startup. <a href=https://github.com/cortexproject/cortex/pull/4271>#4271</a></li><li>[ENHANCEMENT] Ruler: added <code>rule_group</code> label to metrics <code>cortex_prometheus_rule_group_iterations_total</code> and <code>cortex_prometheus_rule_group_iterations_missed_total</code>. <a href=https://github.com/cortexproject/cortex/pull/4121>#4121</a></li><li>[ENHANCEMENT] Ruler: added new metrics for tracking total number of queries and push requests sent to ingester, as well as failed queries and push requests. Failures are only counted for internal errors, but not user-errors like limits or invalid query. This is in contrast to existing <code>cortex_prometheus_rule_evaluation_failures_total</code>, which is incremented also when query or samples appending fails due to user-errors. <a href=https://github.com/cortexproject/cortex/pull/4281>#4281</a><ul><li><code>cortex_ruler_write_requests_total</code></li><li><code>cortex_ruler_write_requests_failed_total</code></li><li><code>cortex_ruler_queries_total</code></li><li><code>cortex_ruler_queries_failed_total</code></li></ul></li><li>[ENHANCEMENT] Ingester: Added option <code>-ingester.ignore-series-limit-for-metric-names</code> with comma-separated list of metric names that will be ignored in max series per metric limit. <a href=https://github.com/cortexproject/cortex/pull/4302>#4302</a></li><li>[ENHANCEMENT] Added instrumentation to Redis client, with the following metrics: <a href=https://github.com/cortexproject/cortex/pull/3976>#3976</a><ul><li><code>cortex_rediscache_request_duration_seconds</code></li></ul></li><li>[BUGFIX] Purger: fix <code>Invalid null value in condition for column range</code> caused by <code>nil</code> value in range for WriteBatch query. <a href=https://github.com/cortexproject/cortex/pull/4128>#4128</a></li><li>[BUGFIX] Ingester: fixed infrequent panic caused by a race condition between TSDB mmap-ed head chunks truncation and queries. <a href=https://github.com/cortexproject/cortex/pull/4176>#4176</a></li><li>[BUGFIX] Alertmanager: fix Alertmanager status page if clustering via gossip is disabled or sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/4184>#4184</a></li><li>[BUGFIX] Ruler: fix <code>/ruler/rule_groups</code> endpoint doesn&rsquo;t work when used with object store. <a href=https://github.com/cortexproject/cortex/pull/4182>#4182</a></li><li>[BUGFIX] Ruler: Honor the evaluation delay for the <code>ALERTS</code> and <code>ALERTS_FOR_STATE</code> series. <a href=https://github.com/cortexproject/cortex/pull/4227>#4227</a></li><li>[BUGFIX] Make multiple Get requests instead of MGet on Redis Cluster. <a href=https://github.com/cortexproject/cortex/pull/4056>#4056</a></li><li>[BUGFIX] Ingester: fix issue where runtime limits erroneously override default limits. <a href=https://github.com/cortexproject/cortex/pull/4246>#4246</a></li><li>[BUGFIX] Ruler: fix startup in single-binary mode when the new <code>ruler_storage</code> is used. <a href=https://github.com/cortexproject/cortex/pull/4252>#4252</a></li><li>[BUGFIX] Querier: fix queries failing with &ldquo;at least 1 healthy replica required, could only find 0&rdquo; error right after scaling up store-gateways until they&rsquo;re ACTIVE in the ring. <a href=https://github.com/cortexproject/cortex/pull/4263>#4263</a></li><li>[BUGFIX] Store-gateway: when blocks sharding is enabled, do not load all blocks in each store-gateway in case of a cold startup, but load only blocks owned by the store-gateway replica. <a href=https://github.com/cortexproject/cortex/pull/4271>#4271</a></li><li>[BUGFIX] Memberlist: fix to setting the default configuration value for <code>-memberlist.retransmit-factor</code> when not provided. This should improve propagation delay of the ring state (including, but not limited to, tombstones). Note that if the configuration is already explicitly given, this fix has no effect. <a href=https://github.com/cortexproject/cortex/pull/4269>#4269</a></li><li>[BUGFIX] Querier: Fix issue where samples in a chunk might get skipped by batch iterator. <a href=https://github.com/cortexproject/cortex/pull/4218>#4218</a></li></ul><h2 id=blocksconvert>Blocksconvert</h2><ul><li>[ENHANCEMENT] Scanner: add support for DynamoDB (v9 schema only). <a href=https://github.com/cortexproject/cortex/pull/3828>#3828</a></li><li>[ENHANCEMENT] Add Cassandra support. <a href=https://github.com/cortexproject/cortex/pull/3795>#3795</a></li><li>[ENHANCEMENT] Scanner: retry failed uploads. <a href=https://github.com/cortexproject/cortex/pull/4188>#4188</a></li></ul><h2 id=190--2021-05-14>1.9.0 / 2021-05-14</h2><ul><li>[CHANGE] Alertmanager now removes local files after Alertmanager is no longer running for removed or resharded user. <a href=https://github.com/cortexproject/cortex/pull/3910>#3910</a></li><li>[CHANGE] Alertmanager now stores local files in per-tenant folders. Files stored by Alertmanager previously are migrated to new hierarchy. Support for this migration will be removed in Cortex 1.11. <a href=https://github.com/cortexproject/cortex/pull/3910>#3910</a></li><li>[CHANGE] Ruler: deprecated <code>-ruler.storage.*</code> CLI flags (and their respective YAML config options) in favour of <code>-ruler-storage.*</code>. The deprecated config will be removed in Cortex 1.11. <a href=https://github.com/cortexproject/cortex/pull/3945>#3945</a></li><li>[CHANGE] Alertmanager: deprecated <code>-alertmanager.storage.*</code> CLI flags (and their respective YAML config options) in favour of <code>-alertmanager-storage.*</code>. This change doesn&rsquo;t apply to <code>alertmanager.storage.path</code> and <code>alertmanager.storage.retention</code>. The deprecated config will be removed in Cortex 1.11. <a href=https://github.com/cortexproject/cortex/pull/4002>#4002</a></li><li>[CHANGE] Alertmanager: removed <code>-cluster.</code> CLI flags deprecated in Cortex 1.7. The new config options to use are: <a href=https://github.com/cortexproject/cortex/pull/3946>#3946</a><ul><li><code>-alertmanager.cluster.listen-address</code> instead of <code>-cluster.listen-address</code></li><li><code>-alertmanager.cluster.advertise-address</code> instead of <code>-cluster.advertise-address</code></li><li><code>-alertmanager.cluster.peers</code> instead of <code>-cluster.peer</code></li><li><code>-alertmanager.cluster.peer-timeout</code> instead of <code>-cluster.peer-timeout</code></li></ul></li><li>[CHANGE] Blocks storage: removed the config option <code>-blocks-storage.bucket-store.index-cache.postings-compression-enabled</code>, which was deprecated in Cortex 1.6. Postings compression is always enabled. <a href=https://github.com/cortexproject/cortex/pull/4101>#4101</a></li><li>[CHANGE] Querier: removed the config option <code>-store.max-look-back-period</code>, which was deprecated in Cortex 1.6 and was used only by the chunks storage. You should use <code>-querier.max-query-lookback</code> instead. <a href=https://github.com/cortexproject/cortex/pull/4101>#4101</a></li><li>[CHANGE] Query Frontend: removed the config option <code>-querier.compress-http-responses</code>, which was deprecated in Cortex 1.6. You should use<code>-api.response-compression-enabled</code> instead. <a href=https://github.com/cortexproject/cortex/pull/4101>#4101</a></li><li>[CHANGE] Runtime-config / overrides: removed the config options <code>-limits.per-user-override-config</code> (use <code>-runtime-config.file</code>) and <code>-limits.per-user-override-period</code> (use <code>-runtime-config.reload-period</code>), both deprecated since Cortex 0.6.0. <a href=https://github.com/cortexproject/cortex/pull/4112>#4112</a></li><li>[CHANGE] Cortex now fails fast on startup if unable to connect to the ring backend. <a href=https://github.com/cortexproject/cortex/pull/4068>#4068</a></li><li>[FEATURE] The following features have been marked as stable: <a href=https://github.com/cortexproject/cortex/pull/4101>#4101</a><ul><li>Shuffle-sharding</li><li>Querier support for querying chunks and blocks store at the same time</li><li>Tracking of active series and exporting them as metrics (<code>-ingester.active-series-metrics-enabled</code> and related flags)</li><li>Blocks storage: lazy mmap of block indexes in the store-gateway (<code>-blocks-storage.bucket-store.index-header-lazy-loading-enabled</code>)</li><li>Ingester: close idle TSDB and remove them from local disk (<code>-blocks-storage.tsdb.close-idle-tsdb-timeout</code>)</li></ul></li><li>[FEATURE] Memberlist: add TLS configuration options for the memberlist transport layer used by the gossip KV store. <a href=https://github.com/cortexproject/cortex/pull/4046>#4046</a><ul><li>New flags added for memberlist communication:<ul><li><code>-memberlist.tls-enabled</code></li><li><code>-memberlist.tls-cert-path</code></li><li><code>-memberlist.tls-key-path</code></li><li><code>-memberlist.tls-ca-path</code></li><li><code>-memberlist.tls-server-name</code></li><li><code>-memberlist.tls-insecure-skip-verify</code></li></ul></li></ul></li><li>[FEATURE] Ruler: added <code>local</code> backend support to the ruler storage configuration under the <code>-ruler-storage.</code> flag prefix. <a href=https://github.com/cortexproject/cortex/pull/3932>#3932</a></li><li>[ENHANCEMENT] Upgraded Docker base images to <code>alpine:3.13</code>. <a href=https://github.com/cortexproject/cortex/pull/4042>#4042</a></li><li>[ENHANCEMENT] Blocks storage: reduce ingester memory by eliminating series reference cache. <a href=https://github.com/cortexproject/cortex/pull/3951>#3951</a></li><li>[ENHANCEMENT] Ruler: optimized <code>&lt;prefix>/api/v1/rules</code> and <code>&lt;prefix>/api/v1/alerts</code> when ruler sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/3916>#3916</a></li><li>[ENHANCEMENT] Ruler: added the following metrics when ruler sharding is enabled: <a href=https://github.com/cortexproject/cortex/pull/3916>#3916</a><ul><li><code>cortex_ruler_clients</code></li><li><code>cortex_ruler_client_request_duration_seconds</code></li></ul></li><li>[ENHANCEMENT] Alertmanager: Add API endpoint to list all tenant alertmanager configs: <code>GET /multitenant_alertmanager/configs</code>. <a href=https://github.com/cortexproject/cortex/pull/3529>#3529</a></li><li>[ENHANCEMENT] Ruler: Add API endpoint to list all tenant ruler rule groups: <code>GET /ruler/rule_groups</code>. <a href=https://github.com/cortexproject/cortex/pull/3529>#3529</a></li><li>[ENHANCEMENT] Query-frontend/scheduler: added querier forget delay (<code>-query-frontend.querier-forget-delay</code> and <code>-query-scheduler.querier-forget-delay</code>) to mitigate the blast radius in the event queriers crash because of a repeatedly sent &ldquo;query of death&rdquo; when shuffle-sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/3901>#3901</a></li><li>[ENHANCEMENT] Query-frontend: reduced memory allocations when serializing query response. <a href=https://github.com/cortexproject/cortex/pull/3964>#3964</a></li><li>[ENHANCEMENT] Querier / ruler: some optimizations to PromQL query engine. <a href=https://github.com/cortexproject/cortex/pull/3934>#3934</a> <a href=https://github.com/cortexproject/cortex/pull/3989>#3989</a></li><li>[ENHANCEMENT] Ingester: reduce CPU and memory when an high number of errors are returned by the ingester on the write path with the blocks storage. <a href=https://github.com/cortexproject/cortex/pull/3969>#3969</a> <a href=https://github.com/cortexproject/cortex/pull/3971>#3971</a> <a href=https://github.com/cortexproject/cortex/pull/3973>#3973</a></li><li>[ENHANCEMENT] Distributor: reduce CPU and memory when an high number of errors are returned by the distributor on the write path. <a href=https://github.com/cortexproject/cortex/pull/3990>#3990</a></li><li>[ENHANCEMENT] Put metric before label value in the &ldquo;label value too long&rdquo; error message. <a href=https://github.com/cortexproject/cortex/pull/4018>#4018</a></li><li>[ENHANCEMENT] Allow use of <code>y|w|d</code> suffixes for duration related limits and per-tenant limits. <a href=https://github.com/cortexproject/cortex/pull/4044>#4044</a></li><li>[ENHANCEMENT] Query-frontend: Small optimization on top of PR <a href=https://github.com/cortexproject/cortex/pull/3968>#3968</a> to avoid unnecessary Extents merging. <a href=https://github.com/cortexproject/cortex/pull/4026>#4026</a></li><li>[ENHANCEMENT] Add a metric <code>cortex_compactor_compaction_interval_seconds</code> for the compaction interval config value. <a href=https://github.com/cortexproject/cortex/pull/4040>#4040</a></li><li>[ENHANCEMENT] Ingester: added following per-ingester (instance) experimental limits: max number of series in memory (<code>-ingester.instance-limits.max-series</code>), max number of users in memory (<code>-ingester.instance-limits.max-tenants</code>), max ingestion rate (<code>-ingester.instance-limits.max-ingestion-rate</code>), and max inflight requests (<code>-ingester.instance-limits.max-inflight-push-requests</code>). These limits are only used when using blocks storage. Limits can also be configured using runtime-config feature, and current values are exported as <code>cortex_ingester_instance_limits</code> metric. <a href=https://github.com/cortexproject/cortex/pull/3992>#3992</a>.</li><li>[ENHANCEMENT] Cortex is now built with Go 1.16. <a href=https://github.com/cortexproject/cortex/pull/4062>#4062</a></li><li>[ENHANCEMENT] Distributor: added per-distributor experimental limits: max number of inflight requests (<code>-distributor.instance-limits.max-inflight-push-requests</code>) and max ingestion rate in samples/sec (<code>-distributor.instance-limits.max-ingestion-rate</code>). If not set, these two are unlimited. Also added metrics to expose current values (<code>cortex_distributor_inflight_push_requests</code>, <code>cortex_distributor_ingestion_rate_samples_per_second</code>) as well as limits (<code>cortex_distributor_instance_limits</code> with various <code>limit</code> label values). <a href=https://github.com/cortexproject/cortex/pull/4071>#4071</a></li><li>[ENHANCEMENT] Ruler: Added <code>-ruler.enabled-tenants</code> and <code>-ruler.disabled-tenants</code> to explicitly enable or disable rules processing for specific tenants. <a href=https://github.com/cortexproject/cortex/pull/4074>#4074</a></li><li>[ENHANCEMENT] Block Storage Ingester: <code>/flush</code> now accepts two new parameters: <code>tenant</code> to specify tenant to flush and <code>wait=true</code> to make call synchronous. Multiple tenants can be specified by repeating <code>tenant</code> parameter. If no <code>tenant</code> is specified, all tenants are flushed, as before. <a href=https://github.com/cortexproject/cortex/pull/4073>#4073</a></li><li>[ENHANCEMENT] Alertmanager: validate configured <code>-alertmanager.web.external-url</code> and fail if ends with <code>/</code>. <a href=https://github.com/cortexproject/cortex/pull/4081>#4081</a></li><li>[ENHANCEMENT] Alertmanager: added <code>-alertmanager.receivers-firewall.block.cidr-networks</code> and <code>-alertmanager.receivers-firewall.block.private-addresses</code> to block specific network addresses in HTTP-based Alertmanager receiver integrations. <a href=https://github.com/cortexproject/cortex/pull/4085>#4085</a></li><li>[ENHANCEMENT] Allow configuration of Cassandra&rsquo;s host selection policy. <a href=https://github.com/cortexproject/cortex/pull/4069>#4069</a></li><li>[ENHANCEMENT] Store-gateway: retry syncing blocks if a per-tenant sync fails. <a href=https://github.com/cortexproject/cortex/pull/3975>#3975</a> <a href=https://github.com/cortexproject/cortex/pull/4088>#4088</a></li><li>[ENHANCEMENT] Add metric <code>cortex_tcp_connections</code> exposing the current number of accepted TCP connections. <a href=https://github.com/cortexproject/cortex/pull/4099>#4099</a></li><li>[ENHANCEMENT] Querier: Allow federated queries to run concurrently. <a href=https://github.com/cortexproject/cortex/pull/4065>#4065</a></li><li>[ENHANCEMENT] Label Values API call now supports <code>match[]</code> parameter when querying blocks on storage (assuming <code>-querier.query-store-for-labels-enabled</code> is enabled). <a href=https://github.com/cortexproject/cortex/pull/4133>#4133</a></li><li>[BUGFIX] Ruler-API: fix bug where <code>/api/v1/rules/&lt;namespace>/&lt;group_name></code> endpoint return <code>400</code> instead of <code>404</code>. <a href=https://github.com/cortexproject/cortex/pull/4013>#4013</a></li><li>[BUGFIX] Distributor: reverted changes done to rate limiting in <a href=https://github.com/cortexproject/cortex/pull/3825>#3825</a>. <a href=https://github.com/cortexproject/cortex/pull/3948>#3948</a></li><li>[BUGFIX] Ingester: Fix race condition when opening and closing tsdb concurrently. <a href=https://github.com/cortexproject/cortex/pull/3959>#3959</a></li><li>[BUGFIX] Querier: streamline tracing spans. <a href=https://github.com/cortexproject/cortex/pull/3924>#3924</a></li><li>[BUGFIX] Ruler Storage: ignore objects with empty namespace or group in the name. <a href=https://github.com/cortexproject/cortex/pull/3999>#3999</a></li><li>[BUGFIX] Distributor: fix issue causing distributors to not extend the replication set because of failing instances when zone-aware replication is enabled. <a href=https://github.com/cortexproject/cortex/pull/3977>#3977</a></li><li>[BUGFIX] Query-frontend: Fix issue where cached entry size keeps increasing when making tiny query repeatedly. <a href=https://github.com/cortexproject/cortex/pull/3968>#3968</a></li><li>[BUGFIX] Compactor: <code>-compactor.blocks-retention-period</code> now supports weeks (<code>w</code>) and years (<code>y</code>). <a href=https://github.com/cortexproject/cortex/pull/4027>#4027</a></li><li>[BUGFIX] Querier: returning 422 (instead of 500) when query hits <code>max_chunks_per_query</code> limit with block storage, when the limit is hit in the store-gateway. <a href=https://github.com/cortexproject/cortex/pull/3937>#3937</a></li><li>[BUGFIX] Ruler: Rule group limit enforcement should now allow the same number of rules in a group as the limit. <a href=https://github.com/cortexproject/cortex/pull/3616>#3616</a></li><li>[BUGFIX] Frontend, Query-scheduler: allow querier to notify about shutdown without providing any authentication. <a href=https://github.com/cortexproject/cortex/pull/4066>#4066</a></li><li>[BUGFIX] Querier: fixed race condition causing queries to fail right after querier startup with the &ldquo;empty ring&rdquo; error. <a href=https://github.com/cortexproject/cortex/pull/4068>#4068</a></li><li>[BUGFIX] Compactor: Increment <code>cortex_compactor_runs_failed_total</code> if compactor failed compact a single tenant. <a href=https://github.com/cortexproject/cortex/pull/4094>#4094</a></li><li>[BUGFIX] Tracing: hot fix to avoid the Jaeger tracing client to indefinitely block the Cortex process shutdown in case the HTTP connection to the tracing backend is blocked. <a href=https://github.com/cortexproject/cortex/pull/4134>#4134</a></li><li>[BUGFIX] Forward proper EndsAt from ruler to Alertmanager inline with Prometheus behaviour. <a href=https://github.com/cortexproject/cortex/pull/4017>#4017</a></li><li>[BUGFIX] Querier: support filtering LabelValues with matchers when using tenant federation. <a href=https://github.com/cortexproject/cortex/pull/4277>#4277</a></li></ul><h2 id=blocksconvert-1>Blocksconvert</h2><ul><li>[ENHANCEMENT] Builder: add <code>-builder.timestamp-tolerance</code> option which may reduce block size by rounding timestamps to make difference whole seconds. <a href=https://github.com/cortexproject/cortex/pull/3891>#3891</a></li></ul><h2 id=181--2021-04-27>1.8.1 / 2021-04-27</h2><ul><li>[CHANGE] Fix for CVE-2021-31232: Local file disclosure vulnerability when <code>-experimental.alertmanager.enable-api</code> is used. The HTTP basic auth <code>password_file</code> can be used as an attack vector to send any file content via a webhook. The alertmanager templates can be used as an attack vector to send any file content because the alertmanager can load any text file specified in the templates list.</li></ul><h2 id=180--2021-03-24>1.8.0 / 2021-03-24</h2><ul><li>[CHANGE] Alertmanager: Don&rsquo;t expose cluster information to tenants via the <code>/alertmanager/api/v1/status</code> API endpoint when operating with clustering enabled. <a href=https://github.com/cortexproject/cortex/pull/3903>#3903</a></li><li>[CHANGE] Ingester: don&rsquo;t update internal &ldquo;last updated&rdquo; timestamp of TSDB if tenant only sends invalid samples. This affects how &ldquo;idle&rdquo; time is computed. <a href=https://github.com/cortexproject/cortex/pull/3727>#3727</a></li><li>[CHANGE] Require explicit flag <code>-&lt;prefix>.tls-enabled</code> to enable TLS in GRPC clients. Previously it was enough to specify a TLS flag to enable TLS validation. <a href=https://github.com/cortexproject/cortex/pull/3156>#3156</a></li><li>[CHANGE] Query-frontend: removed <code>-querier.split-queries-by-day</code> (deprecated in Cortex 0.4.0). Please use <code>-querier.split-queries-by-interval</code> instead. <a href=https://github.com/cortexproject/cortex/pull/3813>#3813</a></li><li>[CHANGE] Store-gateway: the chunks pool controlled by <code>-blocks-storage.bucket-store.max-chunk-pool-bytes</code> is now shared across all tenants. <a href=https://github.com/cortexproject/cortex/pull/3830>#3830</a></li><li>[CHANGE] Ingester: return error code 400 instead of 429 when per-user/per-tenant series/metadata limits are reached. <a href=https://github.com/cortexproject/cortex/pull/3833>#3833</a></li><li>[CHANGE] Compactor: add <code>reason</code> label to <code>cortex_compactor_blocks_marked_for_deletion_total</code> metric. Source blocks marked for deletion by compactor are labelled as <code>compaction</code>, while blocks passing the retention period are labelled as <code>retention</code>. <a href=https://github.com/cortexproject/cortex/pull/3879>#3879</a></li><li>[CHANGE] Alertmanager: the <code>DELETE /api/v1/alerts</code> is now idempotent. No error is returned if the alertmanager config doesn&rsquo;t exist. <a href=https://github.com/cortexproject/cortex/pull/3888>#3888</a></li><li>[FEATURE] Experimental Ruler Storage: Add a separate set of configuration options to configure the ruler storage backend under the <code>-ruler-storage.</code> flag prefix. All blocks storage bucket clients and the config service are currently supported. Clients using this implementation will only be enabled if the existing <code>-ruler.storage</code> flags are left unset. <a href=https://github.com/cortexproject/cortex/pull/3805>#3805</a> <a href=https://github.com/cortexproject/cortex/pull/3864>#3864</a></li><li>[FEATURE] Experimental Alertmanager Storage: Add a separate set of configuration options to configure the alertmanager storage backend under the <code>-alertmanager-storage.</code> flag prefix. All blocks storage bucket clients and the config service are currently supported. Clients using this implementation will only be enabled if the existing <code>-alertmanager.storage</code> flags are left unset. <a href=https://github.com/cortexproject/cortex/pull/3888>#3888</a></li><li>[FEATURE] Adds support to S3 server-side encryption using KMS. The S3 server-side encryption config can be overridden on a per-tenant basis for the blocks storage, ruler and alertmanager. Deprecated <code>-&lt;prefix>.s3.sse-encryption</code>, please use the following CLI flags that have been added. <a href=https://github.com/cortexproject/cortex/pull/3651>#3651</a> <a href=https://github.com/cortexproject/cortex/pull/3810>#3810</a> <a href=https://github.com/cortexproject/cortex/pull/3811>#3811</a> <a href=https://github.com/cortexproject/cortex/pull/3870>#3870</a> <a href=https://github.com/cortexproject/cortex/pull/3886>#3886</a> <a href=https://github.com/cortexproject/cortex/pull/3906>#3906</a><ul><li><code>-&lt;prefix>.s3.sse.type</code></li><li><code>-&lt;prefix>.s3.sse.kms-key-id</code></li><li><code>-&lt;prefix>.s3.sse.kms-encryption-context</code></li></ul></li><li>[FEATURE] Querier: Enable <code>@ &lt;timestamp></code> modifier in PromQL using the new <code>-querier.at-modifier-enabled</code> flag. <a href=https://github.com/cortexproject/cortex/pull/3744>#3744</a></li><li>[FEATURE] Overrides Exporter: Add <code>overrides-exporter</code> module for exposing per-tenant resource limit overrides as metrics. It is not included in <code>all</code> target (single-binary mode), and must be explicitly enabled. <a href=https://github.com/cortexproject/cortex/pull/3785>#3785</a></li><li>[FEATURE] Experimental thanosconvert: introduce an experimental tool <code>thanosconvert</code> to migrate Thanos block metadata to Cortex metadata. <a href=https://github.com/cortexproject/cortex/pull/3770>#3770</a></li><li>[FEATURE] Alertmanager: It now shards the <code>/api/v1/alerts</code> API using the ring when sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/3671>#3671</a><ul><li>Added <code>-alertmanager.max-recv-msg-size</code> (defaults to 16M) to limit the size of HTTP request body handled by the alertmanager.</li><li>New flags added for communication between alertmanagers:<ul><li><code>-alertmanager.max-recv-msg-size</code></li><li><code>-alertmanager.alertmanager-client.remote-timeout</code></li><li><code>-alertmanager.alertmanager-client.tls-enabled</code></li><li><code>-alertmanager.alertmanager-client.tls-cert-path</code></li><li><code>-alertmanager.alertmanager-client.tls-key-path</code></li><li><code>-alertmanager.alertmanager-client.tls-ca-path</code></li><li><code>-alertmanager.alertmanager-client.tls-server-name</code></li><li><code>-alertmanager.alertmanager-client.tls-insecure-skip-verify</code></li></ul></li></ul></li><li>[FEATURE] Compactor: added blocks storage per-tenant retention support. This is configured via <code>-compactor.retention-period</code>, and can be overridden on a per-tenant basis. <a href=https://github.com/cortexproject/cortex/pull/3879>#3879</a></li><li>[ENHANCEMENT] Queries: Instrument queries that were discarded due to the configured <code>max_outstanding_requests_per_tenant</code>. <a href=https://github.com/cortexproject/cortex/pull/3894>#3894</a><ul><li><code>cortex_query_frontend_discarded_requests_total</code></li><li><code>cortex_query_scheduler_discarded_requests_total</code></li></ul></li><li>[ENHANCEMENT] Ruler: Add TLS and explicit basis authentication configuration options for the HTTP client the ruler uses to communicate with the alertmanager. <a href=https://github.com/cortexproject/cortex/pull/3752>#3752</a><ul><li><code>-ruler.alertmanager-client.basic-auth-username</code>: Configure the basic authentication username used by the client. Takes precedent over a URL configured username.</li><li><code>-ruler.alertmanager-client.basic-auth-password</code>: Configure the basic authentication password used by the client. Takes precedent over a URL configured password.</li><li><code>-ruler.alertmanager-client.tls-ca-path</code>: File path to the CA file.</li><li><code>-ruler.alertmanager-client.tls-cert-path</code>: File path to the TLS certificate.</li><li><code>-ruler.alertmanager-client.tls-insecure-skip-verify</code>: Boolean to disable verifying the certificate.</li><li><code>-ruler.alertmanager-client.tls-key-path</code>: File path to the TLS key certificate.</li><li><code>-ruler.alertmanager-client.tls-server-name</code>: Expected name on the TLS certificate.</li></ul></li><li>[ENHANCEMENT] Ingester: exposed metric <code>cortex_ingester_oldest_unshipped_block_timestamp_seconds</code>, tracking the unix timestamp of the oldest TSDB block not shipped to the storage yet. <a href=https://github.com/cortexproject/cortex/pull/3705>#3705</a></li><li>[ENHANCEMENT] Prometheus upgraded. <a href=https://github.com/cortexproject/cortex/pull/3739>#3739</a> <a href=https://github.com/cortexproject/cortex/pull/3806>#3806</a><ul><li>Avoid unnecessary <code>runtime.GC()</code> during compactions.</li><li>Prevent compaction loop in TSDB on data gap.</li></ul></li><li>[ENHANCEMENT] Query-Frontend now returns server side performance metrics using <code>Server-Timing</code> header when query stats is enabled. <a href=https://github.com/cortexproject/cortex/pull/3685>#3685</a></li><li>[ENHANCEMENT] Runtime Config: Add a <code>mode</code> query parameter for the runtime config endpoint. <code>/runtime_config?mode=diff</code> now shows the YAML runtime configuration with all values that differ from the defaults. <a href=https://github.com/cortexproject/cortex/pull/3700>#3700</a></li><li>[ENHANCEMENT] Distributor: Enable downstream projects to wrap distributor push function and access the deserialized write requests before/after they are pushed. <a href=https://github.com/cortexproject/cortex/pull/3755>#3755</a></li><li>[ENHANCEMENT] Add flag <code>-&lt;prefix>.tls-server-name</code> to require a specific server name instead of the hostname on the certificate. <a href=https://github.com/cortexproject/cortex/pull/3156>#3156</a></li><li>[ENHANCEMENT] Alertmanager: Remove a tenant&rsquo;s alertmanager instead of pausing it as we determine it is no longer needed. <a href=https://github.com/cortexproject/cortex/pull/3722>#3722</a></li><li>[ENHANCEMENT] Blocks storage: added more configuration options to S3 client. <a href=https://github.com/cortexproject/cortex/pull/3775>#3775</a><ul><li><code>-blocks-storage.s3.tls-handshake-timeout</code>: Maximum time to wait for a TLS handshake. 0 means no limit.</li><li><code>-blocks-storage.s3.expect-continue-timeout</code>: The time to wait for a server&rsquo;s first response headers after fully writing the request headers if the request has an Expect header. 0 to send the request body immediately.</li><li><code>-blocks-storage.s3.max-idle-connections</code>: Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit.</li><li><code>-blocks-storage.s3.max-idle-connections-per-host</code>: Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used.</li><li><code>-blocks-storage.s3.max-connections-per-host</code>: Maximum number of connections per host. 0 means no limit.</li></ul></li><li>[ENHANCEMENT] Ingester: when tenant&rsquo;s TSDB is closed, Ingester now removes pushed metrics-metadata from memory, and removes metadata (<code>cortex_ingester_memory_metadata</code>, <code>cortex_ingester_memory_metadata_created_total</code>, <code>cortex_ingester_memory_metadata_removed_total</code>) and validation metrics (<code>cortex_discarded_samples_total</code>, <code>cortex_discarded_metadata_total</code>). <a href=https://github.com/cortexproject/cortex/pull/3782>#3782</a></li><li>[ENHANCEMENT] Distributor: cleanup metrics for inactive tenants. <a href=https://github.com/cortexproject/cortex/pull/3784>#3784</a></li><li>[ENHANCEMENT] Ingester: Have ingester to re-emit following TSDB metrics. <a href=https://github.com/cortexproject/cortex/pull/3800>#3800</a><ul><li><code>cortex_ingester_tsdb_blocks_loaded</code></li><li><code>cortex_ingester_tsdb_reloads_total</code></li><li><code>cortex_ingester_tsdb_reloads_failures_total</code></li><li><code>cortex_ingester_tsdb_symbol_table_size_bytes</code></li><li><code>cortex_ingester_tsdb_storage_blocks_bytes</code></li><li><code>cortex_ingester_tsdb_time_retentions_total</code></li></ul></li><li>[ENHANCEMENT] Querier: distribute workload across <code>-store-gateway.sharding-ring.replication-factor</code> store-gateway replicas when querying blocks and <code>-store-gateway.sharding-enabled=true</code>. <a href=https://github.com/cortexproject/cortex/pull/3824>#3824</a></li><li>[ENHANCEMENT] Distributor / HA Tracker: added cleanup of unused elected HA replicas from KV store. Added following metrics to monitor this process: <a href=https://github.com/cortexproject/cortex/pull/3809>#3809</a><ul><li><code>cortex_ha_tracker_replicas_cleanup_started_total</code></li><li><code>cortex_ha_tracker_replicas_cleanup_marked_for_deletion_total</code></li><li><code>cortex_ha_tracker_replicas_cleanup_deleted_total</code></li><li><code>cortex_ha_tracker_replicas_cleanup_delete_failed_total</code></li></ul></li><li>[ENHANCEMENT] Ruler now has new API endpoint <code>/ruler/delete_tenant_config</code> that can be used to delete all ruler groups for tenant. It is intended to be used by administrators who wish to clean up state after removed user. Note that this endpoint is enabled regardless of <code>-experimental.ruler.enable-api</code>. <a href=https://github.com/cortexproject/cortex/pull/3750>#3750</a> <a href=https://github.com/cortexproject/cortex/pull/3899>#3899</a></li><li>[ENHANCEMENT] Query-frontend, query-scheduler: cleanup metrics for inactive tenants. <a href=https://github.com/cortexproject/cortex/pull/3826>#3826</a></li><li>[ENHANCEMENT] Blocks storage: added <code>-blocks-storage.s3.region</code> support to S3 client configuration. <a href=https://github.com/cortexproject/cortex/pull/3811>#3811</a></li><li>[ENHANCEMENT] Distributor: Remove cached subrings for inactive users when using shuffle sharding. <a href=https://github.com/cortexproject/cortex/pull/3849>#3849</a></li><li>[ENHANCEMENT] Store-gateway: Reduced memory used to fetch chunks at query time. <a href=https://github.com/cortexproject/cortex/pull/3855>#3855</a></li><li>[ENHANCEMENT] Ingester: attempt to prevent idle compaction from happening in concurrent ingesters by introducing a 25% jitter to the configured idle timeout (<code>-blocks-storage.tsdb.head-compaction-idle-timeout</code>). <a href=https://github.com/cortexproject/cortex/pull/3850>#3850</a></li><li>[ENHANCEMENT] Compactor: cleanup local files for users that are no longer owned by compactor. <a href=https://github.com/cortexproject/cortex/pull/3851>#3851</a></li><li>[ENHANCEMENT] Store-gateway: close empty bucket stores, and delete leftover local files for tenants that no longer belong to store-gateway. <a href=https://github.com/cortexproject/cortex/pull/3853>#3853</a></li><li>[ENHANCEMENT] Store-gateway: added metrics to track partitioner behaviour. <a href=https://github.com/cortexproject/cortex/pull/3877>#3877</a><ul><li><code>cortex_bucket_store_partitioner_requested_bytes_total</code></li><li><code>cortex_bucket_store_partitioner_requested_ranges_total</code></li><li><code>cortex_bucket_store_partitioner_expanded_bytes_total</code></li><li><code>cortex_bucket_store_partitioner_expanded_ranges_total</code></li></ul></li><li>[ENHANCEMENT] Store-gateway: added metrics to monitor chunk buffer pool behaviour. <a href=https://github.com/cortexproject/cortex/pull/3880>#3880</a><ul><li><code>cortex_bucket_store_chunk_pool_requested_bytes_total</code></li><li><code>cortex_bucket_store_chunk_pool_returned_bytes_total</code></li></ul></li><li>[ENHANCEMENT] Alertmanager: load alertmanager configurations from object storage concurrently, and only load necessary configurations, speeding configuration synchronization process and executing fewer &ldquo;GET object&rdquo; operations to the storage when sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/3898>#3898</a></li><li>[ENHANCEMENT] Ingester (blocks storage): Ingester can now stream entire chunks instead of individual samples to the querier. At the moment this feature must be explicitly enabled either by using <code>-ingester.stream-chunks-when-using-blocks</code> flag or <code>ingester_stream_chunks_when_using_blocks</code> (boolean) field in runtime config file, but these configuration options are temporary and will be removed when feature is stable. <a href=https://github.com/cortexproject/cortex/pull/3889>#3889</a></li><li>[ENHANCEMENT] Alertmanager: New endpoint <code>/multitenant_alertmanager/delete_tenant_config</code> to delete configuration for tenant identified by <code>X-Scope-OrgID</code> header. This is an internal endpoint, available even if Alertmanager API is not enabled by using <code>-experimental.alertmanager.enable-api</code>. <a href=https://github.com/cortexproject/cortex/pull/3900>#3900</a></li><li>[ENHANCEMENT] MemCached: Add <code>max_item_size</code> support. <a href=https://github.com/cortexproject/cortex/pull/3929>#3929</a></li><li>[BUGFIX] Cortex: Fixed issue where fatal errors and various log messages where not logged. <a href=https://github.com/cortexproject/cortex/pull/3778>#3778</a></li><li>[BUGFIX] HA Tracker: don&rsquo;t track as error in the <code>cortex_kv_request_duration_seconds</code> metric a CAS operation intentionally aborted. <a href=https://github.com/cortexproject/cortex/pull/3745>#3745</a></li><li>[BUGFIX] Querier / ruler: do not log &ldquo;error removing stale clients&rdquo; if the ring is empty. <a href=https://github.com/cortexproject/cortex/pull/3761>#3761</a></li><li>[BUGFIX] Store-gateway: fixed a panic caused by a race condition when the index-header lazy loading is enabled. <a href=https://github.com/cortexproject/cortex/pull/3775>#3775</a> <a href=https://github.com/cortexproject/cortex/pull/3789>#3789</a></li><li>[BUGFIX] Compactor: fixed &ldquo;could not guess file size&rdquo; log when uploading blocks deletion marks to the global location. <a href=https://github.com/cortexproject/cortex/pull/3807>#3807</a></li><li>[BUGFIX] Prevent panic at start if the http_prefix setting doesn&rsquo;t have a valid value. <a href=https://github.com/cortexproject/cortex/pull/3796>#3796</a></li><li>[BUGFIX] Memberlist: fixed panic caused by race condition in <code>armon/go-metrics</code> used by memberlist client. <a href=https://github.com/cortexproject/cortex/pull/3725>#3725</a></li><li>[BUGFIX] Querier: returning 422 (instead of 500) when query hits <code>max_chunks_per_query</code> limit with block storage. <a href=https://github.com/cortexproject/cortex/pull/3895>#3895</a></li><li>[BUGFIX] Alertmanager: Ensure that experimental <code>/api/v1/alerts</code> endpoints work when <code>-http.prefix</code> is empty. <a href=https://github.com/cortexproject/cortex/pull/3905>#3905</a></li><li>[BUGFIX] Chunk store: fix panic in inverted index when deleted fingerprint is no longer in the index. <a href=https://github.com/cortexproject/cortex/pull/3543>#3543</a></li></ul><h2 id=171--2021-04-27>1.7.1 / 2021-04-27</h2><ul><li>[CHANGE] Fix for CVE-2021-31232: Local file disclosure vulnerability when <code>-experimental.alertmanager.enable-api</code> is used. The HTTP basic auth <code>password_file</code> can be used as an attack vector to send any file content via a webhook. The alertmanager templates can be used as an attack vector to send any file content because the alertmanager can load any text file specified in the templates list.</li></ul><h2 id=170--2021-02-23>1.7.0 / 2021-02-23</h2><p>Note the blocks storage compactor runs a migration task at startup in this version, which can take many minutes and use a lot of RAM.
+<a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#master--unreleased>master / unreleased</a></li><li><a href=#1181-2024-10-14>1.18.1 2024-10-14</a></li><li><a href=#1180-2024-09-03>1.18.0 2024-09-03</a></li><li><a href=#1171-2024-05-20>1.17.1 2024-05-20</a></li><li><a href=#1170-2024-04-30>1.17.0 2024-04-30</a></li><li><a href=#1161-2024-04-23>1.16.1 2024-04-23</a></li><li><a href=#1160-2023-11-20>1.16.0 2023-11-20</a></li><li><a href=#1153-2023-06-22>1.15.3 2023-06-22</a></li><li><a href=#1152-2023-05-09>1.15.2 2023-05-09</a></li><li><a href=#1151-2023-04-26>1.15.1 2023-04-26</a></li><li><a href=#1150-2023-04-19>1.15.0 2023-04-19</a></li><li><a href=#1140-2022-12-02>1.14.0 2022-12-02</a><ul><li><a href=#known-issues>Known issues</a></li></ul></li><li><a href=#1130-2022-07-14>1.13.0 2022-07-14</a></li><li><a href=#1110-2021-11-25>1.11.0 2021-11-25</a></li><li><a href=#1100--2021-08-03>1.10.0 / 2021-08-03</a></li><li><a href=#blocksconvert>Blocksconvert</a></li><li><a href=#190--2021-05-14>1.9.0 / 2021-05-14</a></li><li><a href=#blocksconvert-1>Blocksconvert</a></li><li><a href=#181--2021-04-27>1.8.1 / 2021-04-27</a></li><li><a href=#180--2021-03-24>1.8.0 / 2021-03-24</a></li><li><a href=#171--2021-04-27>1.7.1 / 2021-04-27</a></li><li><a href=#170--2021-02-23>1.7.0 / 2021-02-23</a></li><li><a href=#160--2020-12-29>1.6.0 / 2020-12-29</a></li><li><a href=#blocksconvert-2>Blocksconvert</a></li><li><a href=#150--2020-11-09>1.5.0 / 2020-11-09</a><ul><li><a href=#cortex>Cortex</a></li><li><a href=#blocksconvert-3>Blocksconvert</a></li></ul></li><li><a href=#140--2020-10-02>1.4.0 / 2020-10-02</a></li><li><a href=#130--2020-08-21>1.3.0 / 2020-08-21</a></li><li><a href=#120--2020-07-01>1.2.0 / 2020-07-01</a></li><li><a href=#110--2020-05-21>1.1.0 / 2020-05-21</a></li><li><a href=#101--2020-04-23>1.0.1 / 2020-04-23</a></li><li><a href=#100--2020-04-02>1.0.0 / 2020-04-02</a><ul><li><a href=#known-issues-1>Known issues</a></li><li><a href=#config-file-breaking-changes>Config file breaking changes</a></li></ul></li><li><a href=#070--2020-03-16>0.7.0 / 2020-03-16</a><ul><li><a href=#full-changelog>Full changelog</a></li><li><a href=#changes-to-denormalised-tokens-in-the-ring>Changes to denormalised tokens in the ring</a></li><li><a href=#known-issues-2>Known issues</a></li><li><a href=#annotated-config-file-breaking-changes>Annotated config file breaking changes</a></li></ul></li><li><a href=#061--2020-02-05>0.6.1 / 2020-02-05</a></li><li><a href=#060--2020-01-28>0.6.0 / 2020-01-28</a><ul><li><a href=#upgrading-postgresql-if-youre-using-configs-service>Upgrading PostgreSQL (if you&rsquo;re using configs service)</a></li><li><a href=#known-issues-3>Known issues</a></li></ul></li><li><a href=#040--2019-12-02>0.4.0 / 2019-12-02</a></li><li><a href=#030--2019-10-11>0.3.0 / 2019-10-11</a></li><li><a href=#020--2019-09-05>0.2.0 / 2019-09-05</a></li><li><a href=#010--2019-08-07>0.1.0 / 2019-08-07</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/changelog/ aria-disabled=true class="btn-link disabled">Changelog</a></li></ol></nav><div class=td-content><h1>Changelog</h1><header class=article-meta></header><h2 id=master--unreleased>master / unreleased</h2><ul><li>[CHANGE] OTLP: Change OTLP handler to be consistent with the Prometheus OTLP handler. <a href=https://github.com/cortexproject/cortex/pull/6272>#6272</a></li></ul><ul><li><code>target_info</code> metric is enabled by default and can be disabled via <code>-distributor.otlp.disable-target-info=true</code> flag</li><li>Convert all attributes to labels is disabled by default and can be enabled via <code>-distributor.otlp.convert-all-attributes=true</code> flag</li><li>You can specify the attributes converted to labels via <code>-distributor.promote-resource-attributes</code> flag. Supported only if <code>-distributor.otlp.convert-all-attributes=false</code></li></ul><ul><li>[CHANGE] Change all max async concurrency default values <code>50</code> to <code>3</code> <a href=https://github.com/cortexproject/cortex/pull/6268>#6268</a></li><li>[CHANGE] Change default value of <code>-blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency</code> from <code>50</code> to <code>3</code> <a href=https://github.com/cortexproject/cortex/pull/6265>#6265</a></li><li>[CHANGE] Enable Compactor and Alertmanager in target all. <a href=https://github.com/cortexproject/cortex/pull/6204>#6204</a></li><li>[FEATURE] Ruler: Pagination support for List Rules API. <a href=https://github.com/cortexproject/cortex/pull/6299>#6299</a></li><li>[FEATURE] Query Frontend/Querier: Add protobuf codec <code>-api.querier-default-codec</code> and the option to choose response compression type <code>-querier.response-compression</code>. <a href=https://github.com/cortexproject/cortex/pull/5527>#5527</a></li><li>[FEATURE] Ruler: Experimental: Add <code>ruler.frontend-address</code> to allow query to query frontends instead of ingesters. <a href=https://github.com/cortexproject/cortex/pull/6151>#6151</a></li><li>[FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via <code>-ruler.enable-ha-evaluation</code> flag. <a href=https://github.com/cortexproject/cortex/pull/6129>#6129</a></li><li>[FEATURE] Store Gateway: Add an in-memory chunk cache. <a href=https://github.com/cortexproject/cortex/pull/6245>#6245</a></li><li>[FEATURE] Chunk Cache: Support multi level cache and add metrics. <a href=https://github.com/cortexproject/cortex/pull/6249>#6249</a></li><li>[FEATURE] Distributor: Accept multiple HA Tracker pairs in the same request. <a href=https://github.com/cortexproject/cortex/pull/6256>#6256</a></li><li>[FEATURE] Ruler: Add support for per-user external labels <a href=https://github.com/cortexproject/cortex/pull/6340>#6340</a></li><li>[ENHANCEMENT] Store Gateway: Add a hedged request to reduce the tail latency. <a href=https://github.com/cortexproject/cortex/pull/6388>#6388</a></li><li>[ENHANCEMENT] Ingester: Add metrics to track succeed/failed native histograms. <a href=https://github.com/cortexproject/cortex/pull/6370>#6370</a></li><li>[ENHANCEMENT] Query Frontend/Querier: Add an experimental flag <code>-querier.enable-promql-experimental-functions</code> to enable experimental promQL functions. <a href=https://github.com/cortexproject/cortex/pull/6355>#6355</a></li><li>[ENHANCEMENT] OTLP: Add <code>-distributor.otlp-max-recv-msg-size</code> flag to limit OTLP request size in bytes. <a href=https://github.com/cortexproject/cortex/pull/6333>#6333</a></li><li>[ENHANCEMENT] S3 Bucket Client: Add a list objects version configs to configure list api object version. <a href=https://github.com/cortexproject/cortex/pull/6280>#6280</a></li><li>[ENHANCEMENT] OpenStack Swift: Add application credential configs for Openstack swift object storage backend. <a href=https://github.com/cortexproject/cortex/pull/6255>#6255</a></li><li>[ENHANCEMENT] Query Frontend: Add new query stats metrics <code>cortex_query_samples_scanned_total</code> and <code>cortex_query_peak_samples</code> to track scannedSamples and peakSample per user. <a href=https://github.com/cortexproject/cortex/pull/6228>#6228</a></li><li>[ENHANCEMENT] Ingester: Add option <code>ingester.disable-chunk-trimming</code> to disable chunk trimming. <a href=https://github.com/cortexproject/cortex/pull/6300>#6300</a></li><li>[ENHANCEMENT] Ingester: Add <code>blocks-storage.tsdb.wal-compression-type</code> to support zstd wal compression type. <a href=https://github.com/cortexproject/cortex/pull/6232>#6232</a></li><li>[ENHANCEMENT] Query Frontend: Add info field to query response. <a href=https://github.com/cortexproject/cortex/pull/6207>#6207</a></li><li>[ENHANCEMENT] Query Frontend: Add peakSample in query stats response. <a href=https://github.com/cortexproject/cortex/pull/6188>#6188</a></li><li>[ENHANCEMENT] Ruler: Add new ruler metric <code>cortex_ruler_rule_groups_in_store</code> that is the total rule groups per tenant in store, which can be used to compare with <code>cortex_prometheus_rule_group_rules</code> to count the number of rule groups that are not loaded by a ruler. <a href=https://github.com/cortexproject/cortex/pull/5869>#5869</a></li><li>[ENHANCEMENT] Ingester/Ring: New <code>READONLY</code> status on ring to be used by Ingester. New ingester API to change mode of ingester <a href=https://github.com/cortexproject/cortex/pull/6163>#6163</a></li><li>[ENHANCEMENT] Ruler: Add query statistics metrics when &ndash;ruler.query-stats-enabled=true. <a href=https://github.com/cortexproject/cortex/pull/6173>#6173</a></li><li>[ENHANCEMENT] Ingester: Add new API <code>/ingester/all_user_stats</code> which shows loaded blocks, active timeseries and ingestion rate for a specific ingester. <a href=https://github.com/cortexproject/cortex/pull/6178>#6178</a></li><li>[ENHANCEMENT] Distributor: Add new <code>cortex_reduced_resolution_histogram_samples_total</code> metric to track the number of histogram samples which resolution was reduced. <a href=https://github.com/cortexproject/cortex/pull/6182>#6182</a></li><li>[ENHANCEMENT] StoreGateway: Implement metadata API limit in queryable. <a href=https://github.com/cortexproject/cortex/pull/6195>#6195</a></li><li>[ENHANCEMENT] Ingester: Add matchers to ingester LabelNames() and LabelNamesStream() RPC. <a href=https://github.com/cortexproject/cortex/pull/6209>#6209</a></li><li>[ENHANCEMENT] KV: Add TLS configs to consul. <a href=https://github.com/cortexproject/cortex/pull/6374>#6374</a></li><li>[ENHANCEMENT] Ingester/Store Gateway Clients: Introduce an experimental HealthCheck handler to quickly fail requests directed to unhealthy targets. <a href=https://github.com/cortexproject/cortex/pull/6225>#6225</a> <a href=https://github.com/cortexproject/cortex/pull/6257>#6257</a></li><li>[ENHANCEMENT] Upgrade build image and Go version to 1.23.2. <a href=https://github.com/cortexproject/cortex/pull/6261>#6261</a> <a href=https://github.com/cortexproject/cortex/pull/6262>#6262</a></li><li>[ENHANCEMENT] Ingester: Introduce a new experimental feature for caching expanded postings on the ingester. <a href=https://github.com/cortexproject/cortex/pull/6296>#6296</a></li><li>[ENHANCEMENT] Querier/Ruler: Expose <code>store_gateway_consistency_check_max_attempts</code> for max retries when querying store gateway in consistency check. <a href=https://github.com/cortexproject/cortex/pull/6276>#6276</a></li><li>[ENHANCEMENT] StoreGateway: Add new <code>cortex_bucket_store_chunk_pool_inuse_bytes</code> metric to track the usage in chunk pool. <a href=https://github.com/cortexproject/cortex/pull/6310>#6310</a></li><li>[ENHANCEMENT] Distributor: Add new <code>cortex_distributor_inflight_client_requests</code> metric to track number of ingester client inflight requests. <a href=https://github.com/cortexproject/cortex/pull/6358>#6358</a></li><li>[ENHANCEMENT] Distributor: Expose <code>cortex_label_size_bytes</code> native histogram metric. <a href=https://github.com/cortexproject/cortex/pull/6372>#6372</a></li><li>[ENHANCEMENT] Add new option <code>-server.grpc_server-num-stream-workers</code> to configure the number of worker goroutines that should be used to process incoming streams. <a href=https://github.com/cortexproject/cortex/pull/6386>#6386</a></li><li>[ENHANCEMENT] Distributor: Return HTTP 5XX instead of HTTP 4XX when instance limits are hit. <a href=https://github.com/cortexproject/cortex/pull/6358>#6358</a></li><li>[ENHANCEMENT] Ingester: Make sure unregistered ingester joining the ring after WAL replay. <a href=https://github.com/cortexproject/cortex/pull/6277>#6277</a></li><li>[BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / <a href=https://github.com/cortexproject/cortex/pull/6224>#6224</a></li><li>[BUGFIX] Ruler: Allow rule evaluation to complete during shutdown. <a href=https://github.com/cortexproject/cortex/pull/6326>#6326</a></li><li>[BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled. <a href=https://github.com/cortexproject/cortex/pull/6271>#6271</a></li><li>[BUGFIX] Ingester: Fix regression on usage of cortex_ingester_queried_chunks. <a href=https://github.com/cortexproject/cortex/pull/6398>#6398</a></li><li>[ENHANCEMENT] Distributor: Add a new <code>-distributor.num-push-workers</code> flag to use a goroutine worker pool when sending data from distributor to ingesters. <a href=https://github.com/cortexproject/cortex/pull/6406>#6406</a></li><li>[BUGFIX] Ingester: Fix possible race condition when <code>active series per LabelSet</code> is configured. <a href=https://github.com/cortexproject/cortex/pull/6409>#6409</a></li></ul><h2 id=1181-2024-10-14>1.18.1 2024-10-14</h2><ul><li>[BUGFIX] Backporting upgrade to go 1.22.7 to patch CVE-2024-34155, CVE-2024-34156, CVE-2024-34158 <a href=https://github.com/cortexproject/cortex/pull/6217>#6217</a> <a href=https://github.com/cortexproject/cortex/pull/6264>#6264</a></li></ul><h2 id=1180-2024-09-03>1.18.0 2024-09-03</h2><ul><li>[CHANGE] Ingester: Remove <code>-querier.query-store-for-labels-enabled</code> flag. Querying long-term store for labels is always enabled. <a href=https://github.com/cortexproject/cortex/pull/5984>#5984</a></li><li>[CHANGE] Server: Instrument <code>cortex_request_duration_seconds</code> metric with native histogram. If <code>native-histograms</code> feature is enabled in monitoring Prometheus then the metric name needs to be updated in your dashboards. <a href=https://github.com/cortexproject/cortex/pull/6056>#6056</a></li><li>[CHANGE] Distributor/Ingester: Change <code>cortex_distributor_ingester_appends_total</code>, <code>cortex_distributor_ingester_append_failures_total</code>, <code>cortex_distributor_ingester_queries_total</code>, and <code>cortex_distributor_ingester_query_failures_total</code> metrics to use the ingester ID instead of its IP as the label value. <a href=https://github.com/cortexproject/cortex/pull/6078>#6078</a></li><li>[CHANGE] OTLP: Set <code>AddMetricSuffixes</code> to true to always enable metric name normalization. <a href=https://github.com/cortexproject/cortex/pull/6136>#6136</a></li><li>[CHANGE] Querier: Deprecate and enable by default <code>querier.ingester-metadata-streaming</code> flag. <a href=https://github.com/cortexproject/cortex/pull/6147>#6147</a></li><li>[CHANGE] QueryFrontend/QueryScheduler: Deprecate <code>-querier.max-outstanding-requests-per-tenant</code> and <code>-query-scheduler.max-outstanding-requests-per-tenant</code> flags. Use frontend.max-outstanding-requests-per-tenant instead. <a href=https://github.com/cortexproject/cortex/pull/6146>#6146</a></li><li>[CHANGE] Ingesters: Enable &lsquo;snappy-block&rsquo; compression on ingester clients by default. <a href=https://github.com/cortexproject/cortex/pull/6148>#6148</a></li><li>[CHANGE] Ruler: Scheduling <code>ruler.evaluation-delay-duration</code> to be deprecated. Ruler will use the highest value between <code>ruler.evaluation-delay-duration</code> and <code>ruler.query-offset</code> <a href=https://github.com/cortexproject/cortex/pull/6149>#6149</a></li><li>[CHANGE] Querier: Remove <code>-querier.at-modifier-enabled</code> flag. <a href=https://github.com/cortexproject/cortex/pull/6157>#6157</a></li><li>[CHANGE] Tracing: Remove deprecated <code>oltp_endpoint</code> config entirely. <a href=https://github.com/cortexproject/cortex/pull/6158>#6158</a></li><li>[CHANGE] Store Gateway: Enable store gateway zone stable shuffle sharding by default. <a href=https://github.com/cortexproject/cortex/pull/6161>#6161</a></li><li>[FEATURE] Ingester/Distributor: Experimental: Enable native histogram ingestion via <code>-blocks-storage.tsdb.enable-native-histograms</code> flag. <a href=https://github.com/cortexproject/cortex/pull/5986>#5986</a> <a href=https://github.com/cortexproject/cortex/pull/6010>#6010</a> <a href=https://github.com/cortexproject/cortex/pull/6020>#6020</a></li><li>[FEATURE] Querier: Enable querying native histogram chunks. <a href=https://github.com/cortexproject/cortex/pull/5944>#5944</a> <a href=https://github.com/cortexproject/cortex/pull/6031>#6031</a></li><li>[FEATURE] Query Frontend: Support native histogram in query frontend response. <a href=https://github.com/cortexproject/cortex/pull/5996>#5996</a> <a href=https://github.com/cortexproject/cortex/pull/6043>#6043</a></li><li>[FEATURE] Ruler: Support sending native histogram samples to Ingester. <a href=https://github.com/cortexproject/cortex/pull/6029>#6029</a></li><li>[FEATURE] Ruler: Add support for filtering out alerts in ListRules API. <a href=https://github.com/cortexproject/cortex/pull/6011>#6011</a></li><li>[FEATURE] Query Frontend: Added a query rejection mechanism to block resource-intensive queries. <a href=https://github.com/cortexproject/cortex/pull/6005>#6005</a></li><li>[FEATURE] OTLP: Support ingesting OTLP exponential metrics as native histograms. <a href=https://github.com/cortexproject/cortex/pull/6071>#6071</a> <a href=https://github.com/cortexproject/cortex/pull/6135>#6135</a></li><li>[FEATURE] Ingester: Add <code>ingester.instance-limits.max-inflight-query-requests</code> to allow limiting ingester concurrent queries. <a href=https://github.com/cortexproject/cortex/pull/6081>#6081</a></li><li>[FEATURE] Distributor: Add <code>validation.max-native-histogram-buckets</code> to limit max number of bucket count. Distributor will try to automatically reduce histogram resolution until it is within the bucket limit or resolution cannot be reduced anymore. <a href=https://github.com/cortexproject/cortex/pull/6104>#6104</a></li><li>[FEATURE] Store Gateway: Introduce token bucket limiter to enhance store gateway throttling. <a href=https://github.com/cortexproject/cortex/pull/6016>#6016</a></li><li>[FEATURE] Ruler: Add support for <code>query_offset</code> field on RuleGroup and new <code>ruler_query_offset</code> per-tenant limit. <a href=https://github.com/cortexproject/cortex/pull/6085>#6085</a></li><li>[ENHANCEMENT] Ruler: Add support to persist tokens in rulers. <a href=https://github.com/cortexproject/cortex/pull/5987>#5987</a></li><li>[ENHANCEMENT] Query Frontend/Querier: Added store gateway postings touched count and touched size in Querier stats and log in Query Frontend. <a href=https://github.com/cortexproject/cortex/pull/5892>#5892</a></li><li>[ENHANCEMENT] Query Frontend/Querier: Returns <code>warnings</code> on prometheus query responses. <a href=https://github.com/cortexproject/cortex/pull/5916>#5916</a></li><li>[ENHANCEMENT] Ingester: Allowing to configure <code>-blocks-storage.tsdb.head-compaction-interval</code> flag up to 30 min and add a jitter on the first head compaction. <a href=https://github.com/cortexproject/cortex/pull/5919>#5919</a> <a href=https://github.com/cortexproject/cortex/pull/5928>#5928</a></li><li>[ENHANCEMENT] Distributor: Added <code>max_inflight_push_requests</code> config to ingester client to protect distributor from OOMKilled. <a href=https://github.com/cortexproject/cortex/pull/5917>#5917</a></li><li>[ENHANCEMENT] Distributor/Querier: Clean stale per-ingester metrics after ingester restarts. <a href=https://github.com/cortexproject/cortex/pull/5930>#5930</a></li><li>[ENHANCEMENT] Distributor/Ring: Allow disabling detailed ring metrics by ring member. <a href=https://github.com/cortexproject/cortex/pull/5931>#5931</a></li><li>[ENHANCEMENT] KV: Etcd Added etcd.ping-without-stream-allowed parameter to disable/enable PermitWithoutStream <a href=https://github.com/cortexproject/cortex/pull/5933>#5933</a></li><li>[ENHANCEMENT] Ingester: Add a new <code>limits_per_label_set</code> limit. This limit functions similarly to <code>max_series_per_metric</code>, but allowing users to define the maximum number of series per LabelSet. <a href=https://github.com/cortexproject/cortex/pull/5950>#5950</a> <a href=https://github.com/cortexproject/cortex/pull/5993>#5993</a></li><li>[ENHANCEMENT] Store Gateway: Log gRPC requests together with headers configured in <code>http_request_headers_to_log</code>. <a href=https://github.com/cortexproject/cortex/pull/5958>#5958</a></li><li>[ENHANCEMENT] Ingester: Add a new experimental <code>-ingester.labels-string-interning-enabled</code> flag to enable string interning for metrics labels. <a href=https://github.com/cortexproject/cortex/pull/6057>#6057</a></li><li>[ENHANCEMENT] Ingester: Add link to renew 10% of the ingesters tokens in the admin page. <a href=https://github.com/cortexproject/cortex/pull/6063>#6063</a></li><li>[ENHANCEMENT] Ruler: Add support for filtering by <code>state</code> and <code>health</code> field on Rules API. <a href=https://github.com/cortexproject/cortex/pull/6040>#6040</a></li><li>[ENHANCEMENT] Ruler: Add support for filtering by <code>match</code> field on Rules API. <a href=https://github.com/cortexproject/cortex/pull/6083>#6083</a></li><li>[ENHANCEMENT] Distributor: Reduce memory usage when error volume is high. <a href=https://github.com/cortexproject/cortex/pull/6095>#6095</a></li><li>[ENHANCEMENT] Compactor: Centralize metrics used by compactor and add user label to compactor metrics. <a href=https://github.com/cortexproject/cortex/pull/6096>#6096</a></li><li>[ENHANCEMENT] Compactor: Add unique execution ID for each compaction cycle in log for easy debugging. <a href=https://github.com/cortexproject/cortex/pull/6097>#6097</a></li><li>[ENHANCEMENT] Compactor: Differentiate retry and halt error and retry failed compaction only on retriable error. <a href=https://github.com/cortexproject/cortex/pull/6111>#6111</a></li><li>[ENHANCEMENT] Ruler: Add support for filtering by <code>state</code> and <code>health</code> field on Rules API. <a href=https://github.com/cortexproject/cortex/pull/6040>#6040</a></li><li>[ENHANCEMENT] Compactor: Split cleaner cycle for active and deleted tenants. <a href=https://github.com/cortexproject/cortex/pull/6112>#6112</a></li><li>[ENHANCEMENT] Compactor: Introduce cleaner visit marker. <a href=https://github.com/cortexproject/cortex/pull/6113>#6113</a></li><li>[ENHANCEMENT] Query Frontend: Add <code>cortex_query_samples_total</code> metric. <a href=https://github.com/cortexproject/cortex/pull/6142>#6142</a></li><li>[ENHANCEMENT] Ingester: Implement metadata API limit. <a href=https://github.com/cortexproject/cortex/pull/6128>#6128</a></li><li>[BUGFIX] Configsdb: Fix endline issue in db password. <a href=https://github.com/cortexproject/cortex/pull/5920>#5920</a></li><li>[BUGFIX] Ingester: Fix <code>user</code> and <code>type</code> labels for the <code>cortex_ingester_tsdb_head_samples_appended_total</code> TSDB metric. <a href=https://github.com/cortexproject/cortex/pull/5952>#5952</a></li><li>[BUGFIX] Querier: Enforce max query length check for <code>/api/v1/series</code> API even though <code>ignoreMaxQueryLength</code> is set to true. <a href=https://github.com/cortexproject/cortex/pull/6018>#6018</a></li><li>[BUGFIX] Ingester: Fix issue with the minimize token generator where it was not taking in consideration the current ownership of an instance when generating extra tokens. <a href=https://github.com/cortexproject/cortex/pull/6062>#6062</a></li><li>[BUGFIX] Scheduler: Fix user queue in scheduler that was not thread-safe. <a href=https://github.com/cortexproject/cortex/pull/6077>#6077</a> <a href=https://github.com/cortexproject/cortex/pull/6160>#6160</a></li><li>[BUGFIX] Ingester: Include out-of-order head compaction when compacting TSDB head. <a href=https://github.com/cortexproject/cortex/pull/6108>#6108</a></li><li>[BUGFIX] Ingester: Fix <code>cortex_ingester_tsdb_mmap_chunks_total</code> metric. <a href=https://github.com/cortexproject/cortex/pull/6134>#6134</a></li><li>[BUGFIX] Query Frontend: Fix query rejection bug for metadata queries. <a href=https://github.com/cortexproject/cortex/pull/6143>#6143</a></li></ul><h2 id=1171-2024-05-20>1.17.1 2024-05-20</h2><ul><li>[CHANGE] Query Frontend/Ruler: Omit empty data, errorType and error fields in API response. <a href=https://github.com/cortexproject/cortex/pull/5953>#5953</a> <a href=https://github.com/cortexproject/cortex/pull/5954>#5954</a></li><li>[ENHANCEMENT] Ingester: Added <code>upload_compacted_blocks_enabled</code> config to ingester to parameterize uploading compacted blocks. <a href=https://github.com/cortexproject/cortex/pull/5959>#5959</a></li><li>[BUGFIX] Querier: Select correct tenant during query federation. <a href=https://github.com/cortexproject/cortex/pull/5943>#5943</a></li></ul><h2 id=1170-2024-04-30>1.17.0 2024-04-30</h2><ul><li>[CHANGE] Azure Storage: Upgraded objstore dependency and support Azure Workload Identity Authentication. Added <code>connection_string</code> to support authenticating via SAS token. Marked <code>msi_resource</code> config as deprecating. <a href=https://github.com/cortexproject/cortex/pull/5645>#5645</a></li><li>[CHANGE] Store Gateway: Add a new fastcache based inmemory index cache. <a href=https://github.com/cortexproject/cortex/pull/5619>#5619</a></li><li>[CHANGE] Index Cache: Multi level cache backfilling operation becomes async. Added <code>-blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency</code> and <code>-blocks-storage.bucket-store.index-cache.multilevel.max-async-buffer-size</code> configs and metric <code>cortex_store_multilevel_index_cache_backfill_dropped_items_total</code> for number of dropped items. <a href=https://github.com/cortexproject/cortex/pull/5661>#5661</a></li><li>[CHANGE] Ingester: Disable uploading compacted blocks and overlapping compaction in ingester. <a href=https://github.com/cortexproject/cortex/pull/5735>#5735</a></li><li>[CHANGE] Distributor: Count the number of rate-limited samples in <code>distributor_samples_in_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5714>#5714</a></li><li>[CHANGE] Ruler: Remove <code>cortex_ruler_write_requests_total</code>, <code>cortex_ruler_write_requests_failed_total</code>, <code>cortex_ruler_queries_total</code>, <code>cortex_ruler_queries_failed_total</code>, and <code>cortex_ruler_query_seconds_total</code> metrics for the tenant when the ruler deletes the manager for the tenant. <a href=https://github.com/cortexproject/cortex/pull/5772>#5772</a></li><li>[CHANGE] Main: Mark <code>mem-ballast-size-bytes</code> flag as deprecated. <a href=https://github.com/cortexproject/cortex/pull/5816>#5816</a></li><li>[CHANGE] Querier: Mark <code>-querier.ingester-streaming</code> flag as deprecated. Now query ingester streaming is always enabled. <a href=https://github.com/cortexproject/cortex/pull/5817>#5817</a></li><li>[CHANGE] Compactor/Bucket Store: Added <code>-blocks-storage.bucket-store.block-discovery-strategy</code> to configure different block listing strategy. Reverted the current recursive block listing mechanism and use the strategy <code>Concurrent</code> as in 1.15. <a href=https://github.com/cortexproject/cortex/pull/5828>#5828</a></li><li>[CHANGE] Compactor: Don&rsquo;t halt compactor when overlapped source blocks detected. <a href=https://github.com/cortexproject/cortex/pull/5854>#5854</a></li><li>[CHANGE] S3 Bucket Client: Expose <code>-blocks-storage.s3.send-content-md5</code> flag and set default checksum algorithm to MD5. <a href=https://github.com/cortexproject/cortex/pull/5870>#5870</a></li><li>[CHANGE] Querier: Mark <code>querier.iterators</code> and <code>querier.batch-iterators</code> flags as deprecated. Now querier always use batch iterators. <a href=https://github.com/cortexproject/cortex/pull/5868>#5868</a></li><li>[CHANGE] Query Frontend: Error response returned by Query Frontend now follows Prometheus API error response format. <a href=https://github.com/cortexproject/cortex/pull/5811>#5811</a></li><li>[FEATURE] Experimental: OTLP ingestion. <a href=https://github.com/cortexproject/cortex/pull/5813>#5813</a></li><li>[FEATURE] Query Frontend/Scheduler: Add query priority support. <a href=https://github.com/cortexproject/cortex/pull/5605>#5605</a></li><li>[FEATURE] Tracing: Use <code>kuberesolver</code> to resolve OTLP endpoints address with <code>kubernetes://</code> prefix as Kubernetes service. <a href=https://github.com/cortexproject/cortex/pull/5731>#5731</a></li><li>[FEATURE] Tracing: Add <code>tracing.otel.round-robin</code> flag to use <code>round_robin</code> gRPC client side LB policy for sending OTLP traces. <a href=https://github.com/cortexproject/cortex/pull/5731>#5731</a></li><li>[FEATURE] Ruler: Add <code>ruler.concurrent-evals-enabled</code> flag to enable concurrent evaluation within a single rule group for independent rules. Maximum concurrency can be configured via <code>ruler.max-concurrent-evals</code>. <a href=https://github.com/cortexproject/cortex/pull/5766>#5766</a></li><li>[FEATURE] Distributor Queryable: Experimental: Add config <code>zone_results_quorum_metadata</code>. When querying ingesters using metadata APIs such as label names and values, only results from quorum number of zones will be included and merged. <a href=https://github.com/cortexproject/cortex/pull/5779>#5779</a></li><li>[FEATURE] Storage Cache Clients: Add config <code>set_async_circuit_breaker_config</code> to utilize the circuit breaker pattern for dynamically thresholding asynchronous set operations. Implemented in both memcached and redis cache clients. <a href=https://github.com/cortexproject/cortex/pull/5789>#5789</a></li><li>[FEATURE] Ruler: Add experimental <code>experimental.ruler.api-deduplicate-rules</code> flag to remove duplicate rule groups from the Prometheus compatible rules API endpoint. Add experimental <code>ruler.ring.replication-factor</code> and <code>ruler.ring.zone-awareness-enabled</code> flags to configure rule group replication, but only the first ruler in the replicaset evaluates the rule group, the rest will be used as backup to handle events when a ruler is down during an API request to list rules. <a href=https://github.com/cortexproject/cortex/pull/5782>#5782</a> <a href=https://github.com/cortexproject/cortex/pull/5901>#5901</a></li><li>[FEATURE] Ring: Add experimental <code>-ingester.tokens-generator-strategy=minimize-spread</code> flag to enable the new minimize spread token generator strategy. <a href=https://github.com/cortexproject/cortex/pull/5855>#5855</a></li><li>[FEATURE] Ring Status Page: Add <code>Ownership Diff From Expected</code> column in the ring table to indicate the extent to which the ownership of a specific ingester differs from the expected ownership. <a href=https://github.com/cortexproject/cortex/pull/5889>#5889</a></li><li>[ENHANCEMENT] Ingester: Add per-tenant new metric <code>cortex_ingester_tsdb_data_replay_duration_seconds</code>. <a href=https://github.com/cortexproject/cortex/pull/5477>#5477</a></li><li>[ENHANCEMENT] Store Gateway: Added <code>-store-gateway.enabled-tenants</code> and <code>-store-gateway.disabled-tenants</code> to explicitly enable or disable store-gateway for specific tenants. <a href=https://github.com/cortexproject/cortex/pull/5638>#5638</a></li><li>[ENHANCEMENT] Query Frontend: Write service timing header in response even though there is an error. <a href=https://github.com/cortexproject/cortex/pull/5653>#5653</a></li><li>[ENHANCEMENT] Compactor: Add new compactor metric <code>cortex_compactor_start_duration_seconds</code>. <a href=https://github.com/cortexproject/cortex/pull/5683>#5683</a></li><li>[ENHANCEMENT] Index Cache: Multi level cache adds config <code>max_backfill_items</code> to cap max items to backfill per async operation. <a href=https://github.com/cortexproject/cortex/pull/5686>#5686</a></li><li>[ENHANCEMENT] Query Frontend: Log number of split queries in <code>query stats</code> log. <a href=https://github.com/cortexproject/cortex/pull/5703>#5703</a></li><li>[ENHANCEMENT] Compactor: Skip compaction retry when encountering a permission denied error. <a href=https://github.com/cortexproject/cortex/pull/5727>#5727</a></li><li>[ENHANCEMENT] Logging: Added new options for logging HTTP request headers: <code>-server.log-request-headers</code> enables logging HTTP request headers, <code>-server.log-request-headers-exclude-list</code> allows users to specify headers which should not be logged. <a href=https://github.com/cortexproject/cortex/pull/5744>#5744</a></li><li>[ENHANCEMENT] Query Frontend/Scheduler: Time check in query priority now considers overall data select time window (including range selectors, modifiers and lookback delta). <a href=https://github.com/cortexproject/cortex/pull/5758>#5758</a></li><li>[ENHANCEMENT] Querier: Added <code>querier.store-gateway-query-stats-enabled</code> to enable or disable store gateway query stats log. <a href=https://github.com/cortexproject/cortex/pull/5749>#5749</a></li><li>[ENHANCEMENT] Querier: Improve labels APIs latency by merging slices using K-way merge and more than 1 core. <a href=https://github.com/cortexproject/cortex/pull/5785>#5785</a></li><li>[ENHANCEMENT] AlertManager: Retrying AlertManager Delete Silence on error. <a href=https://github.com/cortexproject/cortex/pull/5794>#5794</a></li><li>[ENHANCEMENT] Ingester: Add new ingester metric <code>cortex_ingester_max_inflight_query_requests</code>. <a href=https://github.com/cortexproject/cortex/pull/5798>#5798</a></li><li>[ENHANCEMENT] Query: Added <code>query_storage_wall_time</code> to Query Frontend and Ruler query stats log for wall time spent on fetching data from storage. Query evaluation is not included. <a href=https://github.com/cortexproject/cortex/pull/5799>#5799</a></li><li>[ENHANCEMENT] Query: Added additional max query length check at Query Frontend and Ruler. Added <code>-querier.ignore-max-query-length</code> flag to disable max query length check at Querier. <a href=https://github.com/cortexproject/cortex/pull/5808>#5808</a></li><li>[ENHANCEMENT] Querier: Add context error check when converting Metrics to SeriesSet for GetSeries on distributorQuerier. <a href=https://github.com/cortexproject/cortex/pull/5827>#5827</a></li><li>[ENHANCEMENT] Ruler: Improve GetRules response time by reducing lock contention and introducing a temporary rules cache in <code>ruler/manager.go</code>. <a href=https://github.com/cortexproject/cortex/pull/5805>#5805</a></li><li>[ENHANCEMENT] Querier: Add context error check when merging slices from ingesters for GetLabel operations. <a href=https://github.com/cortexproject/cortex/pull/5837>#5837</a></li><li>[BUGFIX] Distributor: Do not use label with empty values for sharding <a href=https://github.com/cortexproject/cortex/pull/5717>#5717</a></li><li>[BUGFIX] Query Frontend: queries with negative offset should check whether it is cacheable or not. <a href=https://github.com/cortexproject/cortex/pull/5719>#5719</a></li><li>[BUGFIX] Redis Cache: pass <code>cache_size</code> config correctly. <a href=https://github.com/cortexproject/cortex/pull/5734>#5734</a></li><li>[BUGFIX] Distributor: Shuffle-Sharding with <code>ingestion_tenant_shard_size</code> set to 0, default sharding strategy should be used. <a href=https://github.com/cortexproject/cortex/pull/5189>#5189</a></li><li>[BUGFIX] Cortex: Fix GRPC stream clients not honoring overrides for call options. <a href=https://github.com/cortexproject/cortex/pull/5797>#5797</a></li><li>[BUGFIX] Ruler: Fix support for <code>keep_firing_for</code> field in alert rules. <a href=https://github.com/cortexproject/cortex/pull/5823>#5823</a></li><li>[BUGFIX] Ring DDB: Fix lifecycle for ring counting unhealthy pods as healthy. <a href=https://github.com/cortexproject/cortex/pull/5838>#5838</a></li><li>[BUGFIX] Ring DDB: Fix region assignment. <a href=https://github.com/cortexproject/cortex/pull/5842>#5842</a></li></ul><h2 id=1161-2024-04-23>1.16.1 2024-04-23</h2><ul><li>[ENHANCEMENT] Upgraded Docker base images to <code>alpine:3.18</code>. <a href=https://github.com/cortexproject/cortex/pull/5684>#5684</a></li><li>[ENHANCEMENT] Upgrade to go 1.21.9 <a href=https://github.com/cortexproject/cortex/pull/5879>#5879</a> <a href=https://github.com/cortexproject/cortex/pull/5882>#5882</a></li></ul><h2 id=1160-2023-11-20>1.16.0 2023-11-20</h2><ul><li>[CHANGE] AlertManager: include reason label in <code>cortex_alertmanager_notifications_failed_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5409>#5409</a></li><li>[CHANGE] Ruler: Added user label to <code>cortex_ruler_write_requests_total</code>, <code>cortex_ruler_write_requests_failed_total</code>, <code>cortex_ruler_queries_total</code>, and <code>cortex_ruler_queries_failed_total</code> metrics. <a href=https://github.com/cortexproject/cortex/pull/5312>#5312</a></li><li>[CHANGE] Alertmanager: Validating new fields on the PagerDuty AM config. <a href=https://github.com/cortexproject/cortex/pull/5290>#5290</a></li><li>[CHANGE] Ingester: Creating label <code>native-histogram-sample</code> on the <code>cortex_discarded_samples_total</code> to keep track of discarded native histogram samples. <a href=https://github.com/cortexproject/cortex/pull/5289>#5289</a></li><li>[CHANGE] Store Gateway: Rename <code>cortex_bucket_store_cached_postings_compression_time_seconds</code> to <code>cortex_bucket_store_cached_postings_compression_time_seconds_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5431>#5431</a></li><li>[CHANGE] Store Gateway: Rename <code>cortex_bucket_store_cached_series_fetch_duration_seconds</code> to <code>cortex_bucket_store_series_fetch_duration_seconds</code> and <code>cortex_bucket_store_cached_postings_fetch_duration_seconds</code> to <code>cortex_bucket_store_postings_fetch_duration_seconds</code>. Add new metric <code>cortex_bucket_store_chunks_fetch_duration_seconds</code>. <a href=https://github.com/cortexproject/cortex/pull/5448>#5448</a></li><li>[CHANGE] Store Gateway: Remove <code>idle_timeout</code>, <code>max_conn_age</code>, <code>pool_size</code>, <code>min_idle_conns</code> fields for Redis index cache and caching bucket. <a href=https://github.com/cortexproject/cortex/pull/5448>#5448</a></li><li>[CHANGE] Store Gateway: Add flag <code>-store-gateway.sharding-ring.zone-stable-shuffle-sharding</code> to enable store gateway to use zone stable shuffle sharding. <a href=https://github.com/cortexproject/cortex/pull/5489>#5489</a></li><li>[CHANGE] Bucket Index: Add <code>series_max_size</code> and <code>chunk_max_size</code> to bucket index. <a href=https://github.com/cortexproject/cortex/pull/5489>#5489</a></li><li>[CHANGE] StoreGateway: Rename <code>cortex_bucket_store_chunk_pool_returned_bytes_total</code> and <code>cortex_bucket_store_chunk_pool_requested_bytes_total</code> to <code>cortex_bucket_store_chunk_pool_operation_bytes_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5552>#5552</a></li><li>[CHANGE] Query Frontend/Querier: Make build info API disabled by default and add feature flag <code>api.build-info-enabled</code> to enable it. <a href=https://github.com/cortexproject/cortex/pull/5533>#5533</a></li><li>[CHANGE] Purger: Do no use S3 tenant kms key when uploading deletion marker. <a href=https://github.com/cortexproject/cortex/pull/5575>#5575</a></li><li>[CHANGE] Ingester: Shipper always allows uploading compacted blocks to ship OOO compacted blocks. <a href=https://github.com/cortexproject/cortex/pull/5625>#5625</a></li><li>[CHANGE] DDBKV: Change metric name from <code>dynamodb_kv_read_capacity_total</code> to <code>dynamodb_kv_consumed_capacity_total</code> and include Delete, Put, Batch dimension. <a href=https://github.com/cortexproject/cortex/pull/5487>#5487</a></li><li>[CHANGE] Compactor: Adding the userId on the compact dir path. <a href=https://github.com/cortexproject/cortex/pull/5524>#5524</a></li><li>[CHANGE] Ingester: Remove deprecated ingester metrics. <a href=https://github.com/cortexproject/cortex/pull/5472>#5472</a></li><li>[CHANGE] Query Frontend: Expose <code>-querier.max-subquery-steps</code> to configure subquery max steps check. By default, the limit is set to 0, which is disabled. <a href=https://github.com/cortexproject/cortex/pull/5656>#5656</a></li><li>[FEATURE] Store Gateway: Implementing multi level index cache. <a href=https://github.com/cortexproject/cortex/pull/5451>#5451</a></li><li>[FEATURE] Ruler: Add support for disabling rule groups. <a href=https://github.com/cortexproject/cortex/pull/5521>#5521</a></li><li>[FEATURE] Support object storage backends for runtime configuration file. <a href=https://github.com/cortexproject/cortex/pull/5292>#5292</a></li><li>[FEATURE] Ruler: Add support for <code>Limit</code> field on RuleGroup. <a href=https://github.com/cortexproject/cortex/pull/5528>#5528</a></li><li>[FEATURE] AlertManager: Add support for Webex, Discord and Telegram Receiver. <a href=https://github.com/cortexproject/cortex/pull/5493>#5493</a></li><li>[FEATURE] Ingester: added <code>-admin-limit-message</code> to customize the message contained in limit errors.<a href=https://github.com/cortexproject/cortex/pull/5460>#5460</a></li><li>[FEATURE] AlertManager: Update version to v0.26.0 and bring in Microsoft Teams receiver. <a href=https://github.com/cortexproject/cortex/pull/5543>#5543</a></li><li>[FEATURE] Store Gateway: Support lazy expanded posting optimization. Added new flag <code>blocks-storage.bucket-store.lazy-expanded-postings-enabled</code> and new metrics <code>cortex_bucket_store_lazy_expanded_postings_total</code>, <code>cortex_bucket_store_lazy_expanded_posting_size_bytes_total</code> and <code>cortex_bucket_store_lazy_expanded_posting_series_overfetched_size_bytes_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5556>#5556</a>.</li><li>[FEATURE] Store Gateway: Add <code>max_downloaded_bytes_per_request</code> to limit max bytes to download per store gateway request. <a href=https://github.com/cortexproject/cortex/pull/5179>#5179</a></li><li>[FEATURE] Added 2 flags <code>-alertmanager.alertmanager-client.grpc-max-send-msg-size</code> and <code>-alertmanager.alertmanager-client.grpc-max-recv-msg-size</code> to configure alert manager grpc client message size limits. <a href=https://github.com/cortexproject/cortex/pull/5338>#5338</a></li><li>[FEATURE] Querier/StoreGateway: Allow the tenant shard sizes to be a percent of total instances. <a href=https://github.com/cortexproject/cortex/pull/5393>#5393</a></li><li>[FEATURE] Added the flag <code>-alertmanager.api-concurrency</code> to configure alert manager api concurrency limit. <a href=https://github.com/cortexproject/cortex/pull/5412>#5412</a></li><li>[FEATURE] Store Gateway: Add <code>-store-gateway.sharding-ring.keep-instance-in-the-ring-on-shutdown</code> to skip unregistering instance from the ring in shutdown. <a href=https://github.com/cortexproject/cortex/pull/5421>#5421</a></li><li>[FEATURE] Ruler: Support for filtering rules in the API. <a href=https://github.com/cortexproject/cortex/pull/5417>#5417</a></li><li>[FEATURE] Compactor: Add <code>-compactor.ring.tokens-file-path</code> to store generated tokens locally. <a href=https://github.com/cortexproject/cortex/pull/5432>#5432</a></li><li>[FEATURE] Query Frontend: Add <code>-frontend.retry-on-too-many-outstanding-requests</code> to re-enqueue 429 requests if there are multiple query-schedulers available. <a href=https://github.com/cortexproject/cortex/pull/5496>#5496</a></li><li>[FEATURE] Store Gateway: Add <code>-blocks-storage.bucket-store.max-inflight-requests</code> for store gateways to reject further series requests upon reaching the limit. <a href=https://github.com/cortexproject/cortex/pull/5553>#5553</a></li><li>[FEATURE] Store Gateway: Support filtered index cache. <a href=https://github.com/cortexproject/cortex/pull/5587>#5587</a></li><li>[ENHANCEMENT] Update go version to 1.21.3. <a href=https://github.com/cortexproject/cortex/pull/5630>#5630</a></li><li>[ENHANCEMENT] Store Gateway: Add <code>cortex_bucket_store_block_load_duration_seconds</code> histogram to track time to load blocks. <a href=https://github.com/cortexproject/cortex/pull/5580>#5580</a></li><li>[ENHANCEMENT] Querier: retry chunk pool exhaustion error in querier rather than query frontend. <a href=https://github.com/cortexproject/cortex/pull/5569>#5569</a></li><li>[ENHANCEMENT] Alertmanager: Added flag <code>-alertmanager.alerts-gc-interval</code> to configure alerts Garbage collection interval. <a href=https://github.com/cortexproject/cortex/pull/5550>#5550</a></li><li>[ENHANCEMENT] Query Frontend: enable vertical sharding on binary expr . <a href=https://github.com/cortexproject/cortex/pull/5507>#5507</a></li><li>[ENHANCEMENT] Query Frontend: Include user agent as part of query frontend log. <a href=https://github.com/cortexproject/cortex/pull/5450>#5450</a></li><li>[ENHANCEMENT] Query: Set CORS Origin headers for Query API <a href=https://github.com/cortexproject/cortex/pull/5388>#5388</a></li><li>[ENHANCEMENT] Query Frontend: Add <code>cortex_rejected_queries_total</code> metric for throttled queries. <a href=https://github.com/cortexproject/cortex/pull/5356>#5356</a></li><li>[ENHANCEMENT] Query Frontend: Optimize the decoding of <code>SampleStream</code>. <a href=https://github.com/cortexproject/cortex/pull/5349>#5349</a></li><li>[ENHANCEMENT] Compactor: Check ctx done when uploading visit marker. <a href=https://github.com/cortexproject/cortex/pull/5333>#5333</a></li><li>[ENHANCEMENT] AlertManager: Add <code>cortex_alertmanager_dispatcher_aggregation_groups</code> and <code>cortex_alertmanager_dispatcher_alert_processing_duration_seconds</code> metrics for dispatcher. <a href=https://github.com/cortexproject/cortex/pull/5592>#5592</a></li><li>[ENHANCEMENT] Store Gateway: Added new flag <code>blocks-storage.bucket-store.series-batch-size</code> to control how many series to fetch per batch in Store Gateway. <a href=https://github.com/cortexproject/cortex/pull/5582>#5582</a>.</li><li>[ENHANCEMENT] Querier: Log query stats when querying store gateway. <a href=https://github.com/cortexproject/cortex/pull/5376>#5376</a></li><li>[ENHANCEMENT] Ruler: Add <code>cortex_ruler_rule_group_load_duration_seconds</code> and <code>cortex_ruler_rule_group_sync_duration_seconds</code> metrics. <a href=https://github.com/cortexproject/cortex/pull/5609>#5609</a></li><li>[ENHANCEMENT] Ruler: Add contextual info and query statistics to log <a href=https://github.com/cortexproject/cortex/pull/5604>#5604</a></li><li>[ENHANCEMENT] Distributor/Ingester: Add span on push path <a href=https://github.com/cortexproject/cortex/pull/5319>#5319</a></li><li>[ENHANCEMENT] Query Frontend: Reject subquery with too small step size. <a href=https://github.com/cortexproject/cortex/pull/5323>#5323</a></li><li>[ENHANCEMENT] Compactor: Exposing Thanos <code>accept-malformed-index</code> to Cortex compactor. <a href=https://github.com/cortexproject/cortex/pull/5334>#5334</a></li><li>[ENHANCEMENT] Log: Avoid expensive <code>log.Valuer</code> evaluation for disallowed levels. <a href=https://github.com/cortexproject/cortex/pull/5297>#5297</a></li><li>[ENHANCEMENT] Improving Performance on the API Gzip Handler. <a href=https://github.com/cortexproject/cortex/pull/5347>#5347</a></li><li>[ENHANCEMENT] Dynamodb: Add <code>puller-sync-time</code> to allow different pull time for ring. <a href=https://github.com/cortexproject/cortex/pull/5357>#5357</a></li><li>[ENHANCEMENT] Emit querier <code>max_concurrent</code> as a metric. <a href=https://github.com/cortexproject/cortex/pull/5362>#5362</a></li><li>[ENHANCEMENT] Avoid sort tokens on lifecycler autoJoin. <a href=https://github.com/cortexproject/cortex/pull/5394>#5394</a></li><li>[ENHANCEMENT] Do not resync blocks in running store gateways during rollout deployment and container restart. <a href=https://github.com/cortexproject/cortex/pull/5363>#5363</a></li><li>[ENHANCEMENT] Store Gateway: Add new metrics <code>cortex_bucket_store_sent_chunk_size_bytes</code>, <code>cortex_bucket_store_postings_size_bytes</code> and <code>cortex_bucket_store_empty_postings_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5397>#5397</a></li><li>[ENHANCEMENT] Add jitter to lifecycler heartbeat. <a href=https://github.com/cortexproject/cortex/pull/5404>#5404</a></li><li>[ENHANCEMENT] Store Gateway: Add config <code>estimated_max_series_size_bytes</code> and <code>estimated_max_chunk_size_bytes</code> to address data overfetch. <a href=https://github.com/cortexproject/cortex/pull/5401>#5401</a></li><li>[ENHANCEMENT] Distributor/Ingester: Add experimental <code>-distributor.sign_write_requests</code> flag to sign the write requests. <a href=https://github.com/cortexproject/cortex/pull/5430>#5430</a></li><li>[ENHANCEMENT] Store Gateway/Querier/Compactor: Handling CMK Access Denied errors. <a href=https://github.com/cortexproject/cortex/pull/5420>#5420</a> <a href=https://github.com/cortexproject/cortex/pull/5442>#5442</a> <a href=https://github.com/cortexproject/cortex/pull/5446>#5446</a></li><li>[ENHANCEMENT] Alertmanager: Add the alert name in error log when it get throttled. <a href=https://github.com/cortexproject/cortex/pull/5456>#5456</a></li><li>[ENHANCEMENT] Querier: Retry store gateway on different zones when zone awareness is enabled. <a href=https://github.com/cortexproject/cortex/pull/5476>#5476</a></li><li>[ENHANCEMENT] Compactor: allow <code>unregister_on_shutdown</code> to be configurable. <a href=https://github.com/cortexproject/cortex/pull/5503>#5503</a></li><li>[ENHANCEMENT] Querier: Batch adding series to query limiter to optimize locking. <a href=https://github.com/cortexproject/cortex/pull/5505>#5505</a></li><li>[ENHANCEMENT] Store Gateway: add metric <code>cortex_bucket_store_chunk_refetches_total</code> for number of chunk refetches. <a href=https://github.com/cortexproject/cortex/pull/5532>#5532</a></li><li>[ENHANCEMENT] BasicLifeCycler: allow final-sleep during shutdown <a href=https://github.com/cortexproject/cortex/pull/5517>#5517</a></li><li>[ENHANCEMENT] All: Handling CMK Access Denied errors. <a href=https://github.com/cortexproject/cortex/pull/5420>#5420</a> <a href=https://github.com/cortexproject/cortex/pull/5542>#5542</a></li><li>[ENHANCEMENT] Querier: Retry store gateway client connection closing gRPC error. <a href=https://github.com/cortexproject/cortex/pull/5558>#5558</a></li><li>[ENHANCEMENT] QueryFrontend: Add generic retry for all APIs. <a href=https://github.com/cortexproject/cortex/pull/5561>#5561</a>.</li><li>[ENHANCEMENT] Querier: Check context before notifying scheduler and frontend. <a href=https://github.com/cortexproject/cortex/pull/5565>#5565</a></li><li>[ENHANCEMENT] QueryFrontend: Add metric for number of series requests. <a href=https://github.com/cortexproject/cortex/pull/5373>#5373</a></li><li>[ENHANCEMENT] Store Gateway: Add histogram metrics for total time spent fetching series and chunks per request. <a href=https://github.com/cortexproject/cortex/pull/5573>#5573</a></li><li>[ENHANCEMENT] Store Gateway: Check context in multi level cache. Add <code>cortex_store_multilevel_index_cache_fetch_duration_seconds</code> and <code>cortex_store_multilevel_index_cache_backfill_duration_seconds</code> to measure fetch and backfill latency. <a href=https://github.com/cortexproject/cortex/pull/5596>#5596</a></li><li>[ENHANCEMENT] Ingester: Added new ingester TSDB metrics <code>cortex_ingester_tsdb_head_samples_appended_total</code>, <code>cortex_ingester_tsdb_head_out_of_order_samples_appended_total</code>, <code>cortex_ingester_tsdb_snapshot_replay_error_total</code>, <code>cortex_ingester_tsdb_sample_ooo_delta</code> and <code>cortex_ingester_tsdb_mmap_chunks_total</code>. <a href=https://github.com/cortexproject/cortex/pull/5624>#5624</a></li><li>[ENHANCEMENT] Query Frontend: Handle context error before decoding and merging responses. <a href=https://github.com/cortexproject/cortex/pull/5499>#5499</a></li><li>[ENHANCEMENT] Store-Gateway and AlertManager: Add a <code>wait_instance_time_out</code> to context to avoid waiting forever. <a href=https://github.com/cortexproject/cortex/pull/5581>#5581</a></li><li>[ENHANCEMENT] Blocks storage: Move the tenant deletion mark from <code>&lt;tenantID>/markers/tenant-deletion-mark.json</code> to <code>__markers__/&lt;tenantID>/tenant-deletion-mark.json</code>. <a href=https://github.com/cortexproject/cortex/pull/5676>#5676</a></li><li>[BUGFIX] Compactor: Fix possible division by zero during compactor config validation. <a href=https://github.com/cortexproject/cortex/pull/5535>#5535</a></li><li>[BUGFIX] Ruler: Validate if rule group can be safely converted back to rule group yaml from protobuf message <a href=https://github.com/cortexproject/cortex/pull/5265>#5265</a></li><li>[BUGFIX] Querier: Convert gRPC <code>ResourceExhausted</code> status code from store gateway to 422 limit error. <a href=https://github.com/cortexproject/cortex/pull/5286>#5286</a></li><li>[BUGFIX] Alertmanager: Route web-ui requests to the alertmanager distributor when sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/5293>#5293</a></li><li>[BUGFIX] Storage: Bucket index updater should ignore meta not found for partial blocks. <a href=https://github.com/cortexproject/cortex/pull/5343>#5343</a></li><li>[BUGFIX] Ring: Add <code>JOINING</code> state to read operation. <a href=https://github.com/cortexproject/cortex/pull/5346>#5346</a></li><li>[BUGFIX] Compactor: Partial block with only visit marker should be deleted even there is no deletion marker. <a href=https://github.com/cortexproject/cortex/pull/5342>#5342</a></li><li>[BUGFIX] KV: Etcd calls will no longer block indefinitely and will now time out after the <code>DialTimeout</code> period. <a href=https://github.com/cortexproject/cortex/pull/5392>#5392</a></li><li>[BUGFIX] Ring: Allow RF greater than number of zones to select more than one instance per zone <a href=https://github.com/cortexproject/cortex/pull/5411>#5411</a></li><li>[BUGFIX] Store Gateway: Fix bug in store gateway ring comparison logic. <a href=https://github.com/cortexproject/cortex/pull/5426>#5426</a></li><li>[BUGFIX] Ring: Fix bug in consistency of Get func in a scaling zone-aware ring. <a href=https://github.com/cortexproject/cortex/pull/5429>#5429</a></li><li>[BUGFIX] Compactor: Fix retry on markers. <a href=https://github.com/cortexproject/cortex/pull/5441>#5441</a></li><li>[BUGFIX] Query Frontend: Fix bug of failing to cancel downstream request context in query frontend v2 mode (query scheduler enabled). <a href=https://github.com/cortexproject/cortex/pull/5447>#5447</a></li><li>[BUGFIX] Alertmanager: Remove the user id from state replication key metric label value. <a href=https://github.com/cortexproject/cortex/pull/5453>#5453</a></li><li>[BUGFIX] Compactor: Avoid cleaner concurrency issues checking global markers before all blocks. <a href=https://github.com/cortexproject/cortex/pull/5457>#5457</a></li><li>[BUGFIX] DDBKV: Disallow instance with older timestamp to update instance with newer timestamp. <a href=https://github.com/cortexproject/cortex/pull/5480>#5480</a></li><li>[BUGFIX] DDBKV: When no change detected in ring, retry the CAS until there is change. <a href=https://github.com/cortexproject/cortex/pull/5502>#5502</a></li><li>[BUGFIX] Fix bug on objstore when configured to use S3 fips endpoints. <a href=https://github.com/cortexproject/cortex/pull/5540>#5540</a></li><li>[BUGFIX] Ruler: Fix bug on ruler where a failure to load a single RuleGroup would prevent rulers to sync all RuleGroup. <a href=https://github.com/cortexproject/cortex/pull/5563>#5563</a></li><li>[BUGFIX] Query Frontend: Fix query string being omitted in query stats log. <a href=https://github.com/cortexproject/cortex/pull/5655>#5655</a></li></ul><h2 id=1153-2023-06-22>1.15.3 2023-06-22</h2><ul><li>[BUGFIX] Distributor: Fix potential data corruption in cases of timeout between distributors and ingesters. <a href=https://github.com/cortexproject/cortex/pull/5422>#5422</a></li></ul><h2 id=1152-2023-05-09>1.15.2 2023-05-09</h2><ul><li>[ENHANCEMENT] Update Go version to 1.20.4. <a href=https://github.com/cortexproject/cortex/pull/5299>#5299</a></li></ul><h2 id=1151-2023-04-26>1.15.1 2023-04-26</h2><ul><li>[CHANGE] Alertmanager: Validating new fields on the PagerDuty AM config. <a href=https://github.com/cortexproject/cortex/pull/5290>#5290</a></li><li>[BUGFIX] Querier: Convert gRPC <code>ResourceExhausted</code> status code from store gateway to 422 limit error. <a href=https://github.com/cortexproject/cortex/pull/5286>#5286</a></li></ul><h2 id=1150-2023-04-19>1.15.0 2023-04-19</h2><ul><li>[CHANGE] Storage: Make Max exemplars config per tenant instead of global configuration. <a href=https://github.com/cortexproject/cortex/pull/5080>#5080</a> <a href=https://github.com/cortexproject/cortex/pull/5122>#5122</a></li><li>[CHANGE] Alertmanager: Local file disclosure vulnerability in OpsGenie configuration has been fixed. <a href=https://github.com/cortexproject/cortex/pull/5045>#5045</a></li><li>[CHANGE] Rename oltp_endpoint to otlp_endpoint to match opentelemetry spec and lib name. <a href=https://github.com/cortexproject/cortex/pull/5068>#5068</a></li><li>[CHANGE] Distributor/Ingester: Log warn level on push requests when they have status code 4xx. Do not log if status is 429. <a href=https://github.com/cortexproject/cortex/pull/5103>#5103</a></li><li>[CHANGE] Tracing: Use the default OTEL trace sampler when <code>-tracing.otel.exporter-type</code> is set to <code>awsxray</code>. <a href=https://github.com/cortexproject/cortex/pull/5141>#5141</a></li><li>[CHANGE] Ingester partial error log line to debug level. <a href=https://github.com/cortexproject/cortex/pull/5192>#5192</a></li><li>[CHANGE] Change HTTP status code from 503/422 to 499 if a request is canceled. <a href=https://github.com/cortexproject/cortex/pull/5220>#5220</a></li><li>[CHANGE] Store gateways summary metrics have been converted to histograms <code>cortex_bucket_store_series_blocks_queried</code>, <code>cortex_bucket_store_series_data_fetched</code>, <code>cortex_bucket_store_series_data_size_touched_bytes</code>, <code>cortex_bucket_store_series_data_size_fetched_bytes</code>, <code>cortex_bucket_store_series_data_touched</code>, <code>cortex_bucket_store_series_result_series</code> <a href=https://github.com/cortexproject/cortex/pull/5239>#5239</a></li><li>[FEATURE] Querier/Query Frontend: support Prometheus /api/v1/status/buildinfo API. <a href=https://github.com/cortexproject/cortex/pull/4978>#4978</a></li><li>[FEATURE] Ingester: Add active series to all_user_stats page. <a href=https://github.com/cortexproject/cortex/pull/4972>#4972</a></li><li>[FEATURE] Ingester: Added <code>-blocks-storage.tsdb.head-chunks-write-queue-size</code> allowing to configure the size of the in-memory queue used before flushing chunks to the disk . <a href=https://github.com/cortexproject/cortex/pull/5000>#5000</a></li><li>[FEATURE] Query Frontend: Log query params in query frontend even if error happens. <a href=https://github.com/cortexproject/cortex/pull/5005>#5005</a></li><li>[FEATURE] Ingester: Enable snapshotting of In-memory TSDB on disk during shutdown via <code>-blocks-storage.tsdb.memory-snapshot-on-shutdown</code>. <a href=https://github.com/cortexproject/cortex/pull/5011>#5011</a></li><li>[FEATURE] Query Frontend/Scheduler: Add a new counter metric <code>cortex_request_queue_requests_total</code> for total requests going to queue. <a href=https://github.com/cortexproject/cortex/pull/5030>#5030</a></li><li>[FEATURE] Build ARM docker images. <a href=https://github.com/cortexproject/cortex/pull/5041>#5041</a></li><li>[FEATURE] Query-frontend/Querier: Create spans to measure time to merge promql responses. <a href=https://github.com/cortexproject/cortex/pull/5041>#5041</a></li><li>[FEATURE] Querier/Ruler: Support the new thanos promql engine. This is an experimental feature and might change in the future. <a href=https://github.com/cortexproject/cortex/pull/5093>#5093</a></li><li>[FEATURE] Added zstd as an option for grpc compression <a href=https://github.com/cortexproject/cortex/pull/5092>#5092</a></li><li>[FEATURE] Ring: Add new kv store option <code>dynamodb</code>. <a href=https://github.com/cortexproject/cortex/pull/5026>#5026</a></li><li>[FEATURE] Cache: Support redis as backend for caching bucket and index cache. <a href=https://github.com/cortexproject/cortex/pull/5057>#5057</a></li><li>[FEATURE] Querier/Store-Gateway: Added <code>-blocks-storage.bucket-store.ignore-blocks-within</code> allowing to filter out the recently created blocks from being synced by queriers and store-gateways. <a href=https://github.com/cortexproject/cortex/pull/5166>#5166</a></li><li>[FEATURE] AlertManager/Ruler: Added support for <code>keep_firing_for</code> on alerting rulers.</li><li>[FEATURE] Alertmanager: Add support for time_intervals. <a href=https://github.com/cortexproject/cortex/pull/5102>#5102</a></li><li>[FEATURE] Added <code>snappy-block</code> as an option for grpc compression <a href=https://github.com/cortexproject/cortex/pull/5215>#5215</a></li><li>[FEATURE] Enable experimental out-of-order samples support. Added 2 new configs <code>ingester.out_of_order_time_window</code> and <code>blocks-storage.tsdb.out_of_order_cap_max</code>. <a href=https://github.com/cortexproject/cortex/pull/4964>#4964</a></li><li>[ENHANCEMENT] Querier: limit series query to only ingesters if <code>start</code> param is not specified. <a href=https://github.com/cortexproject/cortex/pull/4976>#4976</a></li><li>[ENHANCEMENT] Query-frontend/scheduler: add a new limit <code>frontend.max-outstanding-requests-per-tenant</code> for configuring queue size per tenant. Started deprecating two flags <code>-query-scheduler.max-outstanding-requests-per-tenant</code> and <code>-querier.max-outstanding-requests-per-tenant</code>, and change their value default to 0. Now if both the old flag and new flag are specified, the old flag&rsquo;s queue size will be picked. <a href=https://github.com/cortexproject/cortex/pull/4991>#4991</a></li><li>[ENHANCEMENT] Query-tee: Add <code>/api/v1/query_exemplars</code> API endpoint support. <a href=https://github.com/cortexproject/cortex/pull/5010>#5010</a></li><li>[ENHANCEMENT] Let blocks_cleaner delete blocks concurrently(default 16 goroutines). <a href=https://github.com/cortexproject/cortex/pull/5028>#5028</a></li><li>[ENHANCEMENT] Query Frontend/Query Scheduler: Increase upper bound to 60s for queue duration histogram metric. <a href=https://github.com/cortexproject/cortex/pull/5029>#5029</a></li><li>[ENHANCEMENT] Query Frontend: Log Vertical sharding information when <code>query_stats_enabled</code> is enabled. <a href=https://github.com/cortexproject/cortex/pull/5037>#5037</a></li><li>[ENHANCEMENT] Ingester: The metadata APIs should honour <code>querier.query-ingesters-within</code> when <code>querier.query-store-for-labels-enabled</code> is true. <a href=https://github.com/cortexproject/cortex/pull/5027>#5027</a></li><li>[ENHANCEMENT] Query Frontend: Skip instant query roundtripper if sharding is not applicable. <a href=https://github.com/cortexproject/cortex/pull/5062>#5062</a></li><li>[ENHANCEMENT] Push reduce one hash operation of Labels. <a href=https://github.com/cortexproject/cortex/pull/4945>#4945</a> <a href=https://github.com/cortexproject/cortex/pull/5114>#5114</a></li><li>[ENHANCEMENT] Alertmanager: Added <code>-alertmanager.enabled-tenants</code> and <code>-alertmanager.disabled-tenants</code> to explicitly enable or disable alertmanager for specific tenants. <a href=https://github.com/cortexproject/cortex/pull/5116>#5116</a></li><li>[ENHANCEMENT] Upgraded Docker base images to <code>alpine:3.17</code>. <a href=https://github.com/cortexproject/cortex/pull/5132>#5132</a></li><li>[ENHANCEMENT] Add retry logic to S3 bucket client. <a href=https://github.com/cortexproject/cortex/pull/5135>#5135</a></li><li>[ENHANCEMENT] Update Go version to 1.20.1. <a href=https://github.com/cortexproject/cortex/pull/5159>#5159</a></li><li>[ENHANCEMENT] Distributor: Reuse byte slices when serializing requests from distributors to ingesters. <a href=https://github.com/cortexproject/cortex/pull/5193>#5193</a></li><li>[ENHANCEMENT] Query Frontend: Add number of chunks and samples fetched in query stats. <a href=https://github.com/cortexproject/cortex/pull/5198>#5198</a></li><li>[ENHANCEMENT] Implement grpc.Compressor.DecompressedSize for snappy to optimize memory allocations. <a href=https://github.com/cortexproject/cortex/pull/5213>#5213</a></li><li>[ENHANCEMENT] Querier: Batch Iterator optimization to prevent transversing it multiple times query ranges steps does not overlap. <a href=https://github.com/cortexproject/cortex/pull/5237>#5237</a></li><li>[BUGFIX] Updated <code>golang.org/x/net</code> dependency to fix CVE-2022-27664. <a href=https://github.com/cortexproject/cortex/pull/5008>#5008</a></li><li>[BUGFIX] Fix panic when otel and xray tracing is enabled. <a href=https://github.com/cortexproject/cortex/pull/5044>#5044</a></li><li>[BUGFIX] Fixed no compact block got grouped in shuffle sharding grouper. <a href=https://github.com/cortexproject/cortex/pull/5055>#5055</a></li><li>[BUGFIX] Fixed ingesters with less tokens stuck in LEAVING. <a href=https://github.com/cortexproject/cortex/pull/5061>#5061</a></li><li>[BUGFIX] Tracing: Fix missing object storage span instrumentation. <a href=https://github.com/cortexproject/cortex/pull/5074>#5074</a></li><li>[BUGFIX] Ingester: Fix Ingesters returning empty response for metadata APIs. <a href=https://github.com/cortexproject/cortex/pull/5081>#5081</a></li><li>[BUGFIX] Ingester: Fix panic when querying metadata from blocks that are being deleted. <a href=https://github.com/cortexproject/cortex/pull/5119>#5119</a></li><li>[BUGFIX] Ring: Fix case when dynamodb kv reaches the limit of 25 actions per batch call. <a href=https://github.com/cortexproject/cortex/pull/5136>#5136</a></li><li>[BUGFIX] Query-frontend: Fix shardable instant queries do not produce sorted results for <code>sort</code>, <code>sort_desc</code>, <code>topk</code>, <code>bottomk</code> functions. <a href=https://github.com/cortexproject/cortex/pull/5148>#5148</a>, <a href=https://github.com/cortexproject/cortex/pull/5170>#5170</a></li><li>[BUGFIX] Querier: Fix <code>/api/v1/series</code> returning 5XX instead of 4XX when limits are hit. <a href=https://github.com/cortexproject/cortex/pull/5169>#5169</a></li><li>[BUGFIX] Compactor: Fix issue that shuffle sharding planner return error if block is under visit by other compactor. <a href=https://github.com/cortexproject/cortex/pull/5188>#5188</a></li><li>[BUGFIX] Fix S3 BucketWithRetries upload empty content issue <a href=https://github.com/cortexproject/cortex/pull/5217>#5217</a></li><li>[BUGFIX] Query Frontend: Disable <code>absent</code>, <code>absent_over_time</code> and <code>scalar</code> for vertical sharding. <a href=https://github.com/cortexproject/cortex/pull/5221>#5221</a></li><li>[BUGFIX] Catch context error in the s3 bucket client. <a href=https://github.com/cortexproject/cortex/pull/5240>#5240</a></li><li>[BUGFIX] Fix query frontend remote read empty body. <a href=https://github.com/cortexproject/cortex/pull/5257>#5257</a></li><li>[BUGFIX] Fix query frontend incorrect error response format at <code>SplitByQuery</code> middleware. <a href=https://github.com/cortexproject/cortex/pull/5260>#5260</a></li></ul><h2 id=1140-2022-12-02>1.14.0 2022-12-02</h2><p><strong>This release removes support for chunks storage. See below for more.</strong></p><ul><li>[CHANGE] Remove support for chunks storage entirely. If you are using chunks storage on a previous version, you must <a href=https://github.com/cortexproject/cortex/blob/v1.13.1/docs/blocks-storage/migrate-from-chunks-to-blocks.md>migrate your data</a> on version 1.13.1 or earlier. Before upgrading to this release, you should also remove any deprecated chunks-related configuration, as this release will no longer accept that. The following flags are gone:<ul><li><code>-dynamodb.*</code></li><li><code>-metrics.*</code></li><li><code>-s3.*</code></li><li><code>-azure.*</code></li><li><code>-bigtable.*</code></li><li><code>-gcs.*</code></li><li><code>-cassandra.*</code></li><li><code>-boltdb.*</code></li><li><code>-local.*</code></li><li>some <code>-ingester</code> flags:<ul><li><code>-ingester.wal-enabled</code></li><li><code>-ingester.checkpoint-enabled</code></li><li><code>-ingester.recover-from-wal</code></li><li><code>-ingester.wal-dir</code></li><li><code>-ingester.checkpoint-duration</code></li><li><code>-ingester.flush-on-shutdown-with-wal-enabled</code></li><li><code>-ingester.max-transfer-retries</code></li><li><code>-ingester.max-samples-per-query</code></li><li><code>-ingester.min-chunk-length</code></li><li><code>-ingester.flush-period</code></li><li><code>-ingester.retain-period</code></li><li><code>-ingester.max-chunk-idle</code></li><li><code>-ingester.max-stale-chunk-idle</code></li><li><code>-ingester.flush-op-timeout</code></li><li><code>-ingester.max-chunk-age</code></li><li><code>-ingester.chunk-age-jitter</code></li><li><code>-ingester.concurrent-flushes</code></li><li><code>-ingester.spread-flushes</code></li><li><code>-ingester.chunk-encoding</code></li><li><code>-store.*</code> except <code>-store.engine</code> and <code>-store.max-query-length</code></li><li><code>-store.query-chunk-limit</code> was deprecated and replaced by <code>-querier.max-fetched-chunks-per-query</code></li></ul></li><li><code>-deletes.*</code></li><li><code>-grpc-store.*</code></li><li><code>-flusher.wal-dir</code>, <code>-flusher.concurrent-flushes</code>, <code>-flusher.flush-op-timeout</code></li></ul></li><li>[CHANGE] Remove support for alertmanager and ruler legacy store configuration. Before upgrading, you need to convert your configuration to use the <code>alertmanager-storage</code> and <code>ruler-storage</code> configuration on the version that you&rsquo;re already running, then upgrade.</li><li>[CHANGE] Disables TSDB isolation. <a href=https://github.com/cortexproject/cortex/pull/4825>#4825</a></li><li>[CHANGE] Drops support Prometheus 1.x rule format on configdb. <a href=https://github.com/cortexproject/cortex/pull/4826>#4826</a></li><li>[CHANGE] Removes <code>-ingester.stream-chunks-when-using-blocks</code> experimental flag and stream chunks by default when <code>querier.ingester-streaming</code> is enabled. <a href=https://github.com/cortexproject/cortex/pull/4864>#4864</a></li><li>[CHANGE] Compactor: Added <code>cortex_compactor_runs_interrupted_total</code> to separate compaction interruptions from failures</li><li>[CHANGE] Enable PromQL <code>@</code> modifier, negative offset always. <a href=https://github.com/cortexproject/cortex/pull/4927>#4927</a></li><li>[CHANGE] Store-gateway: Add user label to <code>cortex_bucket_store_blocks_loaded</code> metric. <a href=https://github.com/cortexproject/cortex/pull/4918>#4918</a></li><li>[CHANGE] AlertManager: include <code>status</code> label in <code>cortex_alertmanager_alerts_received_total</code>. <a href=https://github.com/cortexproject/cortex/pull/4907>#4907</a></li><li>[FEATURE] Compactor: Added <code>-compactor.block-files-concurrency</code> allowing to configure number of go routines for download/upload block files during compaction. <a href=https://github.com/cortexproject/cortex/pull/4784>#4784</a></li><li>[FEATURE] Compactor: Added <code>-compactor.blocks-fetch-concurrency</code> allowing to configure number of go routines for blocks during compaction. <a href=https://github.com/cortexproject/cortex/pull/4787>#4787</a></li><li>[FEATURE] Compactor: Added configurations for Azure MSI in blocks-storage, ruler-storage and alertmanager-storage. <a href=https://github.com/cortexproject/cortex/pull/4818>#4818</a></li><li>[FEATURE] Ruler: Add support to pass custom implementations of queryable and pusher. <a href=https://github.com/cortexproject/cortex/pull/4782>#4782</a></li><li>[FEATURE] Create OpenTelemetry Bridge for Tracing. Now cortex can send traces to multiple destinations using OTEL Collectors. <a href=https://github.com/cortexproject/cortex/pull/4834>#4834</a></li><li>[FEATURE] Added <code>-api.http-request-headers-to-log</code> allowing for the addition of HTTP Headers to logs <a href=https://github.com/cortexproject/cortex/pull/4803>#4803</a></li><li>[FEATURE] Distributor: Added a new limit <code>-validation.max-labels-size-bytes</code> allowing to limit the combined size of labels for each timeseries. <a href=https://github.com/cortexproject/cortex/pull/4848>#4848</a></li><li>[FEATURE] Storage/Bucket: Added <code>-*.s3.bucket-lookup-type</code> allowing to configure the s3 bucket lookup type. <a href=https://github.com/cortexproject/cortex/pull/4794>#4794</a></li><li>[FEATURE] QueryFrontend: Implement experimental vertical sharding at query frontend for range/instant queries. <a href=https://github.com/cortexproject/cortex/pull/4863>#4863</a></li><li>[FEATURE] QueryFrontend: Support vertical sharding for subqueries. <a href=https://github.com/cortexproject/cortex/pull/4955>#4955</a></li><li>[FEATURE] Querier: Added a new limit <code>-querier.max-fetched-data-bytes-per-query</code> allowing to limit the maximum size of all data in bytes that a query can fetch from each ingester and storage. <a href=https://github.com/cortexproject/cortex/pull/4854>#4854</a></li><li>[FEATURE] Added 2 flags <code>-alertmanager.alertmanager-client.grpc-compression</code> and <code>-querier.store-gateway-client.grpc-compression</code> to configure compression methods for grpc clients. <a href=https://github.com/cortexproject/cortex/pull/4889>#4889</a></li><li>[ENHANCEMENT] AlertManager: Retrying AlertManager Get Requests (Get Alertmanager status, Get Alertmanager Receivers) on next replica on error <a href=https://github.com/cortexproject/cortex/pull/4840>#4840</a></li><li>[ENHANCEMENT] Querier/Ruler: Retry store-gateway in case of unexpected failure, instead of failing the query. <a href=https://github.com/cortexproject/cortex/pull/4532>#4532</a> <a href=https://github.com/cortexproject/cortex/pull/4839>#4839</a></li><li>[ENHANCEMENT] Ring: DoBatch prioritize 4xx errors when failing. <a href=https://github.com/cortexproject/cortex/pull/4783>#4783</a></li><li>[ENHANCEMENT] Cortex now built with Go 1.18. <a href=https://github.com/cortexproject/cortex/pull/4829>#4829</a></li><li>[ENHANCEMENT] Ingester: Prevent ingesters to become unhealthy during wall replay. <a href=https://github.com/cortexproject/cortex/pull/4847>#4847</a></li><li>[ENHANCEMENT] Compactor: Introduced visit marker file for blocks so blocks are under compaction will not be picked up by another compactor. <a href=https://github.com/cortexproject/cortex/pull/4805>#4805</a></li><li>[ENHANCEMENT] Distributor: Add label name to labelValueTooLongError. <a href=https://github.com/cortexproject/cortex/pull/4855>#4855</a></li><li>[ENHANCEMENT] Enhance traces with hostname information. <a href=https://github.com/cortexproject/cortex/pull/4898>#4898</a></li><li>[ENHANCEMENT] Improve the documentation around limits. <a href=https://github.com/cortexproject/cortex/pull/4905>#4905</a></li><li>[ENHANCEMENT] Distributor: cache user overrides to reduce lock contention. <a href=https://github.com/cortexproject/cortex/pull/4904>#4904</a></li><li>[BUGFIX] Storage/Bucket: Enable AWS SDK for go authentication for s3 to fix IMDSv1 authentication. <a href=https://github.com/cortexproject/cortex/pull/4897>#4897</a></li><li>[BUGFIX] Memberlist: Add join with no retrying when starting service. <a href=https://github.com/cortexproject/cortex/pull/4804>#4804</a></li><li>[BUGFIX] Ruler: Fix /ruler/rule_groups returns YAML with extra fields. <a href=https://github.com/cortexproject/cortex/pull/4767>#4767</a></li><li>[BUGFIX] Respecting <code>-tracing.otel.sample-ratio</code> configuration when enabling OpenTelemetry tracing with X-ray. <a href=https://github.com/cortexproject/cortex/pull/4862>#4862</a></li><li>[BUGFIX] QueryFrontend: fixed query_range requests when query has <code>start</code> equals to <code>end</code>. <a href=https://github.com/cortexproject/cortex/pull/4877>#4877</a></li><li>[BUGFIX] AlertManager: fixed issue introduced by <a href=https://github.com/cortexproject/cortex/pull/4495>#4495</a> where templates files were being deleted when using alertmanager local store. <a href=https://github.com/cortexproject/cortex/pull/4890>#4890</a></li><li>[BUGFIX] Ingester: fixed incorrect logging at the start of ingester block shipping logic. <a href=https://github.com/cortexproject/cortex/pull/4934>#4934</a></li><li>[BUGFIX] Storage/Bucket: fixed global mark missing on deletion. <a href=https://github.com/cortexproject/cortex/pull/4949>#4949</a></li><li>[BUGFIX] QueryFrontend/Querier: fixed regression added by <a href=https://github.com/cortexproject/cortex/pull/4863>#4863</a> where we stopped compressing the response between querier and query frontend. <a href=https://github.com/cortexproject/cortex/pull/4960>#4960</a></li><li>[BUGFIX] QueryFrontend/Querier: fixed fix response error to be ungzipped when status code is not 2xx. <a href=https://github.com/cortexproject/cortex/pull/4975>#4975</a></li></ul><h3 id=known-issues>Known issues</h3><ul><li>Configsdb: Ruler configs doesn&rsquo;t work. Remove all configs from postgres database that have format Prometheus 1.x rule format before upgrading to v1.14.0 (see <a href=https://github.com/cortexproject/cortex/issues/5387>5387</a>)</li></ul><h2 id=1130-2022-07-14>1.13.0 2022-07-14</h2><ul><li>[CHANGE] Changed default for <code>-ingester.min-ready-duration</code> from 1 minute to 15 seconds. <a href=https://github.com/cortexproject/cortex/pull/4539>#4539</a></li><li>[CHANGE] query-frontend: Do not print anything in the logs of <code>query-frontend</code> if a in-progress query has been canceled (context canceled) to avoid spam. <a href=https://github.com/cortexproject/cortex/pull/4562>#4562</a></li><li>[CHANGE] Compactor block deletion mark migration, needed when upgrading from v1.7, is now disabled by default. <a href=https://github.com/cortexproject/cortex/pull/4597>#4597</a></li><li>[CHANGE] The <code>status_code</code> label on gRPC client metrics has changed from &lsquo;200&rsquo; and &lsquo;500&rsquo; to &lsquo;2xx&rsquo;, &lsquo;5xx&rsquo;, &lsquo;4xx&rsquo;, &lsquo;cancel&rsquo; or &rsquo;error&rsquo;. <a href=https://github.com/cortexproject/cortex/pull/4601>#4601</a></li><li>[CHANGE] Memberlist: changed probe interval from <code>1s</code> to <code>5s</code> and probe timeout from <code>500ms</code> to <code>2s</code>. <a href=https://github.com/cortexproject/cortex/pull/4601>#4601</a></li><li>[CHANGE] Fix incorrectly named <code>cortex_cache_fetched_keys</code> and <code>cortex_cache_hits</code> metrics. Renamed to <code>cortex_cache_fetched_keys_total</code> and <code>cortex_cache_hits_total</code> respectively. <a href=https://github.com/cortexproject/cortex/pull/4686>#4686</a></li><li>[CHANGE] Enable Thanos series limiter in store-gateway. <a href=https://github.com/cortexproject/cortex/pull/4702>#4702</a></li><li>[CHANGE] Distributor: Apply <code>max_fetched_series_per_query</code> limit for <code>/series</code> API. <a href=https://github.com/cortexproject/cortex/pull/4683>#4683</a></li><li>[CHANGE] Re-enable the <code>proxy_url</code> option for receiver configuration. <a href=https://github.com/cortexproject/cortex/pull/4741>#4741</a></li><li>[FEATURE] Ruler: Add <code>external_labels</code> option to tag all alerts with a given set of labels. <a href=https://github.com/cortexproject/cortex/pull/4499>#4499</a></li><li>[FEATURE] Compactor: Add <code>-compactor.skip-blocks-with-out-of-order-chunks-enabled</code> configuration to mark blocks containing index with out-of-order chunks for no compact instead of halting the compaction. <a href=https://github.com/cortexproject/cortex/pull/4707>#4707</a></li><li>[FEATURE] Querier/Query-Frontend: Add <code>-querier.per-step-stats-enabled</code> and <code>-frontend.cache-queryable-samples-stats</code> configurations to enable query sample statistics. <a href=https://github.com/cortexproject/cortex/pull/4708>#4708</a></li><li>[FEATURE] Add shuffle sharding for the compactor <a href=https://github.com/cortexproject/cortex/pull/4433>#4433</a></li><li>[FEATURE] Querier: Use streaming for ingester metadata APIs. <a href=https://github.com/cortexproject/cortex/pull/4725>#4725</a></li><li>[ENHANCEMENT] Update Go version to 1.17.8. <a href=https://github.com/cortexproject/cortex/pull/4602>#4602</a> <a href=https://github.com/cortexproject/cortex/pull/4604>#4604</a> <a href=https://github.com/cortexproject/cortex/pull/4658>#4658</a></li><li>[ENHANCEMENT] Keep track of discarded samples due to bad relabel configuration in <code>cortex_discarded_samples_total</code>. <a href=https://github.com/cortexproject/cortex/pull/4503>#4503</a></li><li>[ENHANCEMENT] Ruler: Add <code>-ruler.disable-rule-group-label</code> to disable the <code>rule_group</code> label on exported metrics. <a href=https://github.com/cortexproject/cortex/pull/4571>#4571</a></li><li>[ENHANCEMENT] Query federation: improve performance in MergeQueryable by memoizing labels. <a href=https://github.com/cortexproject/cortex/pull/4502>#4502</a></li><li>[ENHANCEMENT] Added new ring related config <code>-ingester.readiness-check-ring-health</code> when enabled the readiness probe will succeed only after all instances are ACTIVE and healthy in the ring, this is enabled by default. <a href=https://github.com/cortexproject/cortex/pull/4539>#4539</a></li><li>[ENHANCEMENT] Added new ring related config <code>-distributor.excluded-zones</code> when set this will exclude the comma-separated zones from the ring, default is &ldquo;&rdquo;. <a href=https://github.com/cortexproject/cortex/pull/4539>#4539</a></li><li>[ENHANCEMENT] Upgraded Docker base images to <code>alpine:3.14</code>. <a href=https://github.com/cortexproject/cortex/pull/4514>#4514</a></li><li>[ENHANCEMENT] Updated Prometheus to latest. Includes changes from prometheus<a href=https://github.com/cortexproject/cortex/pull/9239>#9239</a>, adding 15 new functions. Multiple TSDB bugfixes prometheus<a href=https://github.com/cortexproject/cortex/pull/9438>#9438</a> & prometheus<a href=https://github.com/cortexproject/cortex/pull/9381>#9381</a>. <a href=https://github.com/cortexproject/cortex/pull/4524>#4524</a></li><li>[ENHANCEMENT] Query Frontend: Add setting <code>-frontend.forward-headers-list</code> in frontend to configure the set of headers from the requests to be forwarded to downstream requests. <a href=https://github.com/cortexproject/cortex/pull/4486>#4486</a></li><li>[ENHANCEMENT] Blocks storage: Add <code>-blocks-storage.azure.http.*</code>, <code>-alertmanager-storage.azure.http.*</code>, and <code>-ruler-storage.azure.http.*</code> to configure the Azure storage client. <a href=https://github.com/cortexproject/cortex/pull/4581>#4581</a></li><li>[ENHANCEMENT] Optimise memberlist receive path when used as a backing store for rings with a large number of members. <a href=https://github.com/cortexproject/cortex/pull/4601>#4601</a></li><li>[ENHANCEMENT] Add length and limit to labelNameTooLongError and labelValueTooLongError <a href=https://github.com/cortexproject/cortex/pull/4595>#4595</a></li><li>[ENHANCEMENT] Add jitter to rejoinInterval. <a href=https://github.com/cortexproject/cortex/pull/4747>#4747</a></li><li>[ENHANCEMENT] Compactor: uploading blocks no compaction marks to the global location and introduce a new metric <a href=https://github.com/cortexproject/cortex/pull/4729>#4729</a><ul><li><code>cortex_bucket_blocks_marked_for_no_compaction_count</code>: Total number of blocks marked for no compaction in the bucket.</li></ul></li><li>[ENHANCEMENT] Querier: Reduce the number of series that are kept in memory while streaming from ingesters. <a href=https://github.com/cortexproject/cortex/pull/4745>#4745</a></li><li>[BUGFIX] AlertManager: remove stale template files. <a href=https://github.com/cortexproject/cortex/pull/4495>#4495</a></li><li>[BUGFIX] Distributor: fix bug in query-exemplar where some results would get dropped. <a href=https://github.com/cortexproject/cortex/pull/4583>#4583</a></li><li>[BUGFIX] Update Thanos dependency: compactor tracing support, azure blocks storage memory fix. <a href=https://github.com/cortexproject/cortex/pull/4585>#4585</a></li><li>[BUGFIX] Set appropriate <code>Content-Type</code> header for /services endpoint, which previously hard-coded <code>text/plain</code>. <a href=https://github.com/cortexproject/cortex/pull/4596>#4596</a></li><li>[BUGFIX] Querier: Disable query scheduler SRV DNS lookup, which removes noisy log messages about &ldquo;failed DNS SRV record lookup&rdquo;. <a href=https://github.com/cortexproject/cortex/pull/4601>#4601</a></li><li>[BUGFIX] Memberlist: fixed corrupted packets when sending compound messages with more than 255 messages or messages bigger than 64KB. <a href=https://github.com/cortexproject/cortex/pull/4601>#4601</a></li><li>[BUGFIX] Query Frontend: If &lsquo;LogQueriesLongerThan&rsquo; is set to &lt; 0, log all queries as described in the docs. <a href=https://github.com/cortexproject/cortex/pull/4633>#4633</a></li><li>[BUGFIX] Distributor: update defaultReplicationStrategy to not fail with extend-write when a single instance is unhealthy. <a href=https://github.com/cortexproject/cortex/pull/4636>#4636</a></li><li>[BUGFIX] Distributor: Fix race condition on <code>/series</code> introduced by <a href=https://github.com/cortexproject/cortex/pull/4683>#4683</a>. <a href=https://github.com/cortexproject/cortex/pull/4716>#4716</a></li><li>[BUGFIX] Ruler: Fixed leaking notifiers after users are removed <a href=https://github.com/cortexproject/cortex/pull/4718>#4718</a></li><li>[BUGFIX] Distributor: Fix a memory leak in distributor due to the cluster label. <a href=https://github.com/cortexproject/cortex/pull/4739>#4739</a></li><li>[BUGFIX] Memberlist: Avoid clock skew by limiting the timestamp accepted on gossip. <a href=https://github.com/cortexproject/cortex/pull/4750>#4750</a></li><li>[BUGFIX] Compactor: skip compaction if there is only 1 block available for shuffle-sharding compactor. <a href=https://github.com/cortexproject/cortex/pull/4756>#4756</a></li><li>[BUGFIX] Compactor: Fixes <a href=https://github.com/cortexproject/cortex/pull/4770>#4770</a> - an edge case in compactor with shulffle sharding where compaction stops when a tenant stops ingesting samples. <a href=https://github.com/cortexproject/cortex/pull/4771>#4771</a></li><li>[BUGFIX] Compactor: fix cortex_compactor_remaining_planned_compactions not set after plan generation for shuffle sharding compactor. <a href=https://github.com/cortexproject/cortex/pull/4772>#4772</a></li></ul><h2 id=1110-2021-11-25>1.11.0 2021-11-25</h2><ul><li>[CHANGE] Memberlist: Expose default configuration values to the command line options. Note that setting these explicitly to zero will no longer cause the default to be used. If the default is desired, then do set the option. The following are affected: <a href=https://github.com/cortexproject/cortex/pull/4276>#4276</a><ul><li><code>-memberlist.stream-timeout</code></li><li><code>-memberlist.retransmit-factor</code></li><li><code>-memberlist.pull-push-interval</code></li><li><code>-memberlist.gossip-interval</code></li><li><code>-memberlist.gossip-nodes</code></li><li><code>-memberlist.gossip-to-dead-nodes-time</code></li><li><code>-memberlist.dead-node-reclaim-time</code></li></ul></li><li>[CHANGE] <code>-querier.max-fetched-chunks-per-query</code> previously applied to chunks from ingesters and store separately; now the two combined should not exceed the limit. <a href=https://github.com/cortexproject/cortex/pull/4260>#4260</a></li><li>[CHANGE] Memberlist: the metric <code>memberlist_kv_store_value_bytes</code> has been removed due to values no longer being stored in-memory as encoded bytes. <a href=https://github.com/cortexproject/cortex/pull/4345>#4345</a></li><li>[CHANGE] Some files and directories created by Cortex components on local disk now have stricter permissions, and are only readable by owner, but not group or others. <a href=https://github.com/cortexproject/cortex/pull/4394>#4394</a></li><li>[CHANGE] The metric <code>cortex_deprecated_flags_inuse_total</code> has been renamed to <code>deprecated_flags_inuse_total</code> as part of using grafana/dskit functionality. <a href=https://github.com/cortexproject/cortex/pull/4443>#4443</a></li><li>[FEATURE] Ruler: Add new <code>-ruler.query-stats-enabled</code> which when enabled will report the <code>cortex_ruler_query_seconds_total</code> as a per-user metric that tracks the sum of the wall time of executing queries in the ruler in seconds. <a href=https://github.com/cortexproject/cortex/pull/4317>#4317</a></li><li>[FEATURE] Query Frontend: Add <code>cortex_query_fetched_series_total</code> and <code>cortex_query_fetched_chunks_bytes_total</code> per-user counters to expose the number of series and bytes fetched as part of queries. These metrics can be enabled with the <code>-frontend.query-stats-enabled</code> flag (or its respective YAML config option <code>query_stats_enabled</code>). <a href=https://github.com/cortexproject/cortex/pull/4343>#4343</a></li><li>[FEATURE] AlertManager: Add support for SNS Receiver. <a href=https://github.com/cortexproject/cortex/pull/4382>#4382</a></li><li>[FEATURE] Distributor: Add label <code>status</code> to metric <code>cortex_distributor_ingester_append_failures_total</code> <a href=https://github.com/cortexproject/cortex/pull/4442>#4442</a></li><li>[FEATURE] Queries: Added <code>present_over_time</code> PromQL function, also some TSDB optimisations. <a href=https://github.com/cortexproject/cortex/pull/4505>#4505</a></li><li>[ENHANCEMENT] Add timeout for waiting on compactor to become ACTIVE in the ring. <a href=https://github.com/cortexproject/cortex/pull/4262>#4262</a></li><li>[ENHANCEMENT] Reduce memory used by streaming queries, particularly in ruler. <a href=https://github.com/cortexproject/cortex/pull/4341>#4341</a></li><li>[ENHANCEMENT] Ring: allow experimental configuration of disabling of heartbeat timeouts by setting the relevant configuration value to zero. Applies to the following: <a href=https://github.com/cortexproject/cortex/pull/4342>#4342</a><ul><li><code>-distributor.ring.heartbeat-timeout</code></li><li><code>-ring.heartbeat-timeout</code></li><li><code>-ruler.ring.heartbeat-timeout</code></li><li><code>-alertmanager.sharding-ring.heartbeat-timeout</code></li><li><code>-compactor.ring.heartbeat-timeout</code></li><li><code>-store-gateway.sharding-ring.heartbeat-timeout</code></li></ul></li><li>[ENHANCEMENT] Ring: allow heartbeats to be explicitly disabled by setting the interval to zero. This is considered experimental. This applies to the following configuration options: <a href=https://github.com/cortexproject/cortex/pull/4344>#4344</a><ul><li><code>-distributor.ring.heartbeat-period</code></li><li><code>-ingester.heartbeat-period</code></li><li><code>-ruler.ring.heartbeat-period</code></li><li><code>-alertmanager.sharding-ring.heartbeat-period</code></li><li><code>-compactor.ring.heartbeat-period</code></li><li><code>-store-gateway.sharding-ring.heartbeat-period</code></li></ul></li><li>[ENHANCEMENT] Memberlist: optimized receive path for processing ring state updates, to help reduce CPU utilization in large clusters. <a href=https://github.com/cortexproject/cortex/pull/4345>#4345</a></li><li>[ENHANCEMENT] Memberlist: expose configuration of memberlist packet compression via <code>-memberlist.compression=enabled</code>. <a href=https://github.com/cortexproject/cortex/pull/4346>#4346</a></li><li>[ENHANCEMENT] Update Go version to 1.16.6. <a href=https://github.com/cortexproject/cortex/pull/4362>#4362</a></li><li>[ENHANCEMENT] Updated Prometheus to include changes from prometheus/prometheus<a href=https://github.com/cortexproject/cortex/pull/9083>#9083</a>. Now whenever <code>/labels</code> API calls include matchers, blocks store is queried for <code>LabelNames</code> with matchers instead of <code>Series</code> calls which was inefficient. <a href=https://github.com/cortexproject/cortex/pull/4380>#4380</a></li><li>[ENHANCEMENT] Querier: performance improvements in socket and memory handling. <a href=https://github.com/cortexproject/cortex/pull/4429>#4429</a> <a href=https://github.com/cortexproject/cortex/pull/4377>#4377</a></li><li>[ENHANCEMENT] Exemplars are now emitted for all gRPC calls and many operations tracked by histograms. <a href=https://github.com/cortexproject/cortex/pull/4462>#4462</a></li><li>[ENHANCEMENT] New options <code>-server.http-listen-network</code> and <code>-server.grpc-listen-network</code> allow binding as &rsquo;tcp4&rsquo; or &rsquo;tcp6&rsquo;. <a href=https://github.com/cortexproject/cortex/pull/4462>#4462</a></li><li>[ENHANCEMENT] Rulers: Using shuffle sharding subring on GetRules API. <a href=https://github.com/cortexproject/cortex/pull/4466>#4466</a></li><li>[ENHANCEMENT] Support memcached auto-discovery via <code>auto-discovery</code> flag, introduced by thanos in <a href=https://github.com/thanos-io/thanos/pull/4487>https://github.com/thanos-io/thanos/pull/4487</a>. Both AWS and Google Cloud memcached service support auto-discovery, which returns a list of nodes of the memcached cluster. <a href=https://github.com/cortexproject/cortex/pull/4412>#4412</a></li><li>[BUGFIX] Fixes a panic in the query-tee when comparing result. <a href=https://github.com/cortexproject/cortex/pull/4465>#4465</a></li><li>[BUGFIX] Frontend: Fixes @ modifier functions (start/end) when splitting queries by time. <a href=https://github.com/cortexproject/cortex/pull/4464>#4464</a></li><li>[BUGFIX] Compactor: compactor will no longer try to compact blocks that are already marked for deletion. Previously compactor would consider blocks marked for deletion within <code>-compactor.deletion-delay / 2</code> period as eligible for compaction. <a href=https://github.com/cortexproject/cortex/pull/4328>#4328</a></li><li>[BUGFIX] HA Tracker: when cleaning up obsolete elected replicas from KV store, tracker didn&rsquo;t update number of cluster per user correctly. <a href=https://github.com/cortexproject/cortex/pull/4336>#4336</a></li><li>[BUGFIX] Ruler: fixed counting of PromQL evaluation errors as user-errors when updating <code>cortex_ruler_queries_failed_total</code>. <a href=https://github.com/cortexproject/cortex/pull/4335>#4335</a></li><li>[BUGFIX] Ingester: When using block storage, prevent any reads or writes while the ingester is stopping. This will prevent accessing TSDB blocks once they have been already closed. <a href=https://github.com/cortexproject/cortex/pull/4304>#4304</a></li><li>[BUGFIX] Ingester: fixed ingester stuck on start up (LEAVING ring state) when <code>-ingester.heartbeat-period=0</code> and <code>-ingester.unregister-on-shutdown=false</code>. <a href=https://github.com/cortexproject/cortex/pull/4366>#4366</a></li><li>[BUGFIX] Ingester: panic during shutdown while fetching batches from cache. <a href=https://github.com/cortexproject/cortex/pull/4397>#4397</a></li><li>[BUGFIX] Querier: After query-frontend restart, querier may have lower than configured concurrency. <a href=https://github.com/cortexproject/cortex/pull/4417>#4417</a></li><li>[BUGFIX] Memberlist: forward only changes, not entire original message. <a href=https://github.com/cortexproject/cortex/pull/4419>#4419</a></li><li>[BUGFIX] Memberlist: don&rsquo;t accept old tombstones as incoming change, and don&rsquo;t forward such messages to other gossip members. <a href=https://github.com/cortexproject/cortex/pull/4420>#4420</a></li><li>[BUGFIX] Querier: fixed panic when querying exemplars and using <code>-distributor.shard-by-all-labels=false</code>. <a href=https://github.com/cortexproject/cortex/pull/4473>#4473</a></li><li>[BUGFIX] Querier: honor querier minT,maxT if <code>nil</code> SelectHints are passed to Select(). <a href=https://github.com/cortexproject/cortex/pull/4413>#4413</a></li><li>[BUGFIX] Compactor: fixed panic while collecting Prometheus metrics. <a href=https://github.com/cortexproject/cortex/pull/4483>#4483</a></li><li>[BUGFIX] Update go-kit package to fix spurious log messages <a href=https://github.com/cortexproject/cortex/pull/4544>#4544</a></li></ul><h2 id=1100--2021-08-03>1.10.0 / 2021-08-03</h2><ul><li>[CHANGE] Prevent path traversal attack from users able to control the HTTP header <code>X-Scope-OrgID</code>. <a href=https://github.com/cortexproject/cortex/pull/4375>#4375</a> (CVE-2021-36157)<ul><li>Users only have control of the HTTP header when Cortex is not frontend by an auth proxy validating the tenant IDs</li></ul></li><li>[CHANGE] Enable strict JSON unmarshal for <code>pkg/util/validation.Limits</code> struct. The custom <code>UnmarshalJSON()</code> will now fail if the input has unknown fields. <a href=https://github.com/cortexproject/cortex/pull/4298>#4298</a></li><li>[CHANGE] Cortex chunks storage has been deprecated and it&rsquo;s now in maintenance mode: all Cortex users are encouraged to migrate to the blocks storage. No new features will be added to the chunks storage. The default Cortex configuration still runs the chunks engine; please check out the <a href=https://cortexmetrics.io/docs/blocks-storage/>blocks storage doc</a> on how to configure Cortex to run with the blocks storage. <a href=https://github.com/cortexproject/cortex/pull/4268>#4268</a></li><li>[CHANGE] The example Kubernetes manifests (stored at <code>k8s/</code>) have been removed due to a lack of proper support and maintenance. <a href=https://github.com/cortexproject/cortex/pull/4268>#4268</a></li><li>[CHANGE] Querier / ruler: deprecated <code>-store.query-chunk-limit</code> CLI flag (and its respective YAML config option <code>max_chunks_per_query</code>) in favour of <code>-querier.max-fetched-chunks-per-query</code> (and its respective YAML config option <code>max_fetched_chunks_per_query</code>). The new limit specifies the maximum number of chunks that can be fetched in a single query from ingesters and long-term storage: the total number of actual fetched chunks could be 2x the limit, being independently applied when querying ingesters and long-term storage. <a href=https://github.com/cortexproject/cortex/pull/4125>#4125</a></li><li>[CHANGE] Alertmanager: allowed to configure the experimental receivers firewall on a per-tenant basis. The following CLI flags (and their respective YAML config options) have been changed and moved to the limits config section: <a href=https://github.com/cortexproject/cortex/pull/4143>#4143</a><ul><li><code>-alertmanager.receivers-firewall.block.cidr-networks</code> renamed to <code>-alertmanager.receivers-firewall-block-cidr-networks</code></li><li><code>-alertmanager.receivers-firewall.block.private-addresses</code> renamed to <code>-alertmanager.receivers-firewall-block-private-addresses</code></li></ul></li><li>[CHANGE] Change default value of <code>-server.grpc.keepalive.min-time-between-pings</code> from <code>5m</code> to <code>10s</code> and <code>-server.grpc.keepalive.ping-without-stream-allowed</code> to <code>true</code>. <a href=https://github.com/cortexproject/cortex/pull/4168>#4168</a></li><li>[CHANGE] Ingester: Change default value of <code>-ingester.active-series-metrics-enabled</code> to <code>true</code>. This incurs a small increase in memory usage, between 1.2% and 1.6% as measured on ingesters with 1.3M active series. <a href=https://github.com/cortexproject/cortex/pull/4257>#4257</a></li><li>[CHANGE] Dependency: update go-redis from v8.2.3 to v8.9.0. <a href=https://github.com/cortexproject/cortex/pull/4236>#4236</a></li><li>[FEATURE] Querier: Added new <code>-querier.max-fetched-series-per-query</code> flag. When Cortex is running with blocks storage, the max series per query limit is enforced in the querier and applies to unique series received from ingesters and store-gateway (long-term storage). <a href=https://github.com/cortexproject/cortex/pull/4179>#4179</a></li><li>[FEATURE] Querier/Ruler: Added new <code>-querier.max-fetched-chunk-bytes-per-query</code> flag. When Cortex is running with blocks storage, the max chunk bytes limit is enforced in the querier and ruler and limits the size of all aggregated chunks returned from ingesters and storage as bytes for a query. <a href=https://github.com/cortexproject/cortex/pull/4216>#4216</a></li><li>[FEATURE] Alertmanager: support negative matchers, time-based muting - <a href=https://github.com/prometheus/alertmanager/releases/tag/v0.22.0>upstream release notes</a>. <a href=https://github.com/cortexproject/cortex/pull/4237>#4237</a></li><li>[FEATURE] Alertmanager: Added rate-limits to notifiers. Rate limits used by all integrations can be configured using <code>-alertmanager.notification-rate-limit</code>, while per-integration rate limits can be specified via <code>-alertmanager.notification-rate-limit-per-integration</code> parameter. Both shared and per-integration limits can be overwritten using overrides mechanism. These limits are applied on individual (per-tenant) alertmanagers. Rate-limited notifications are failed notifications. It is possible to monitor rate-limited notifications via new <code>cortex_alertmanager_notification_rate_limited_total</code> metric. <a href=https://github.com/cortexproject/cortex/pull/4135>#4135</a> <a href=https://github.com/cortexproject/cortex/pull/4163>#4163</a></li><li>[FEATURE] Alertmanager: Added <code>-alertmanager.max-config-size-bytes</code> limit to control size of configuration files that Cortex users can upload to Alertmanager via API. This limit is configurable per-tenant. <a href=https://github.com/cortexproject/cortex/pull/4201>#4201</a></li><li>[FEATURE] Alertmanager: Added <code>-alertmanager.max-templates-count</code> and <code>-alertmanager.max-template-size-bytes</code> options to control number and size of templates uploaded to Alertmanager via API. These limits are configurable per-tenant. <a href=https://github.com/cortexproject/cortex/pull/4223>#4223</a></li><li>[FEATURE] Added flag <code>-debug.block-profile-rate</code> to enable goroutine blocking events profiling. <a href=https://github.com/cortexproject/cortex/pull/4217>#4217</a></li><li>[FEATURE] Alertmanager: The experimental sharding feature is now considered complete. Detailed information about the configuration options can be found <a href=https://cortexmetrics.io/docs/configuration/configuration-file/#alertmanager_config>here for alertmanager</a> and <a href=https://cortexmetrics.io/docs/configuration/configuration-file/#alertmanager_storage_config>here for the alertmanager storage</a>. To use the feature: <a href=https://github.com/cortexproject/cortex/pull/3925>#3925</a> <a href=https://github.com/cortexproject/cortex/pull/4020>#4020</a> <a href=https://github.com/cortexproject/cortex/pull/4021>#4021</a> <a href=https://github.com/cortexproject/cortex/pull/4031>#4031</a> <a href=https://github.com/cortexproject/cortex/pull/4084>#4084</a> <a href=https://github.com/cortexproject/cortex/pull/4110>#4110</a> <a href=https://github.com/cortexproject/cortex/pull/4126>#4126</a> <a href=https://github.com/cortexproject/cortex/pull/4127>#4127</a> <a href=https://github.com/cortexproject/cortex/pull/4141>#4141</a> <a href=https://github.com/cortexproject/cortex/pull/4146>#4146</a> <a href=https://github.com/cortexproject/cortex/pull/4161>#4161</a> <a href=https://github.com/cortexproject/cortex/pull/4162>#4162</a> <a href=https://github.com/cortexproject/cortex/pull/4222>#4222</a><ul><li>Ensure that a remote storage backend is configured for Alertmanager to store state using <code>-alertmanager-storage.backend</code>, and flags related to the backend. Note that the <code>local</code> and <code>configdb</code> storage backends are not supported.</li><li>Ensure that a ring store is configured using <code>-alertmanager.sharding-ring.store</code>, and set the flags relevant to the chosen store type.</li><li>Enable the feature using <code>-alertmanager.sharding-enabled</code>.</li><li>Note the prior addition of a new configuration option <code>-alertmanager.persist-interval</code>. This sets the interval between persisting the current alertmanager state (notification log and silences) to object storage. See the <a href=https://cortexmetrics.io/docs/configuration/configuration-file/#alertmanager_config>configuration file reference</a> for more information.</li></ul></li><li>[ENHANCEMENT] Alertmanager: Cleanup persisted state objects from remote storage when a tenant configuration is deleted. <a href=https://github.com/cortexproject/cortex/pull/4167>#4167</a></li><li>[ENHANCEMENT] Storage: Added the ability to disable Open Census within GCS client (e.g <code>-gcs.enable-opencensus=false</code>). <a href=https://github.com/cortexproject/cortex/pull/4219>#4219</a></li><li>[ENHANCEMENT] Etcd: Added username and password to etcd config. <a href=https://github.com/cortexproject/cortex/pull/4205>#4205</a></li><li>[ENHANCEMENT] Alertmanager: introduced new metrics to monitor operation when using <code>-alertmanager.sharding-enabled</code>: <a href=https://github.com/cortexproject/cortex/pull/4149>#4149</a><ul><li><code>cortex_alertmanager_state_fetch_replica_state_total</code></li><li><code>cortex_alertmanager_state_fetch_replica_state_failed_total</code></li><li><code>cortex_alertmanager_state_initial_sync_total</code></li><li><code>cortex_alertmanager_state_initial_sync_completed_total</code></li><li><code>cortex_alertmanager_state_initial_sync_duration_seconds</code></li><li><code>cortex_alertmanager_state_persist_total</code></li><li><code>cortex_alertmanager_state_persist_failed_total</code></li></ul></li><li>[ENHANCEMENT] Blocks storage: support ingesting exemplars and querying of exemplars. Enabled by setting new CLI flag <code>-blocks-storage.tsdb.max-exemplars=&lt;n></code> or config option <code>blocks_storage.tsdb.max_exemplars</code> to positive value. <a href=https://github.com/cortexproject/cortex/pull/4124>#4124</a> <a href=https://github.com/cortexproject/cortex/pull/4181>#4181</a></li><li>[ENHANCEMENT] Distributor: Added distributors ring status section in the admin page. <a href=https://github.com/cortexproject/cortex/pull/4151>#4151</a></li><li>[ENHANCEMENT] Added zone-awareness support to alertmanager for use when sharding is enabled. When zone-awareness is enabled, alerts will be replicated across availability zones. <a href=https://github.com/cortexproject/cortex/pull/4204>#4204</a></li><li>[ENHANCEMENT] Added <code>tenant_ids</code> tag to tracing spans <a href=https://github.com/cortexproject/cortex/pull/4186>#4186</a></li><li>[ENHANCEMENT] Ring, query-frontend: Avoid using automatic private IPs (APIPA) when discovering IP address from the interface during the registration of the instance in the ring, or by query-frontend when used with query-scheduler. APIPA still used as last resort with logging indicating usage. <a href=https://github.com/cortexproject/cortex/pull/4032>#4032</a></li><li>[ENHANCEMENT] Memberlist: introduced new metrics to aid troubleshooting tombstone convergence: <a href=https://github.com/cortexproject/cortex/pull/4231>#4231</a><ul><li><code>memberlist_client_kv_store_value_tombstones</code></li><li><code>memberlist_client_kv_store_value_tombstones_removed_total</code></li><li><code>memberlist_client_messages_to_broadcast_dropped_total</code></li></ul></li><li>[ENHANCEMENT] Alertmanager: Added <code>-alertmanager.max-dispatcher-aggregation-groups</code> option to control max number of active dispatcher groups in Alertmanager (per tenant, also overridable). When the limit is reached, Dispatcher produces log message and increases <code>cortex_alertmanager_dispatcher_aggregation_group_limit_reached_total</code> metric. <a href=https://github.com/cortexproject/cortex/pull/4254>#4254</a></li><li>[ENHANCEMENT] Alertmanager: Added <code>-alertmanager.max-alerts-count</code> and <code>-alertmanager.max-alerts-size-bytes</code> to control max number of alerts and total size of alerts that a single user can have in Alertmanager&rsquo;s memory. Adding more alerts will fail with a log message and incrementing <code>cortex_alertmanager_alerts_insert_limited_total</code> metric (per-user). These limits can be overrided by using per-tenant overrides. Current values are tracked in <code>cortex_alertmanager_alerts_limiter_current_alerts</code> and <code>cortex_alertmanager_alerts_limiter_current_alerts_size_bytes</code> metrics. <a href=https://github.com/cortexproject/cortex/pull/4253>#4253</a></li><li>[ENHANCEMENT] Store-gateway: added <code>-store-gateway.sharding-ring.wait-stability-min-duration</code> and <code>-store-gateway.sharding-ring.wait-stability-max-duration</code> support to store-gateway, to wait for ring stability at startup. <a href=https://github.com/cortexproject/cortex/pull/4271>#4271</a></li><li>[ENHANCEMENT] Ruler: added <code>rule_group</code> label to metrics <code>cortex_prometheus_rule_group_iterations_total</code> and <code>cortex_prometheus_rule_group_iterations_missed_total</code>. <a href=https://github.com/cortexproject/cortex/pull/4121>#4121</a></li><li>[ENHANCEMENT] Ruler: added new metrics for tracking total number of queries and push requests sent to ingester, as well as failed queries and push requests. Failures are only counted for internal errors, but not user-errors like limits or invalid query. This is in contrast to existing <code>cortex_prometheus_rule_evaluation_failures_total</code>, which is incremented also when query or samples appending fails due to user-errors. <a href=https://github.com/cortexproject/cortex/pull/4281>#4281</a><ul><li><code>cortex_ruler_write_requests_total</code></li><li><code>cortex_ruler_write_requests_failed_total</code></li><li><code>cortex_ruler_queries_total</code></li><li><code>cortex_ruler_queries_failed_total</code></li></ul></li><li>[ENHANCEMENT] Ingester: Added option <code>-ingester.ignore-series-limit-for-metric-names</code> with comma-separated list of metric names that will be ignored in max series per metric limit. <a href=https://github.com/cortexproject/cortex/pull/4302>#4302</a></li><li>[ENHANCEMENT] Added instrumentation to Redis client, with the following metrics: <a href=https://github.com/cortexproject/cortex/pull/3976>#3976</a><ul><li><code>cortex_rediscache_request_duration_seconds</code></li></ul></li><li>[BUGFIX] Purger: fix <code>Invalid null value in condition for column range</code> caused by <code>nil</code> value in range for WriteBatch query. <a href=https://github.com/cortexproject/cortex/pull/4128>#4128</a></li><li>[BUGFIX] Ingester: fixed infrequent panic caused by a race condition between TSDB mmap-ed head chunks truncation and queries. <a href=https://github.com/cortexproject/cortex/pull/4176>#4176</a></li><li>[BUGFIX] Alertmanager: fix Alertmanager status page if clustering via gossip is disabled or sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/4184>#4184</a></li><li>[BUGFIX] Ruler: fix <code>/ruler/rule_groups</code> endpoint doesn&rsquo;t work when used with object store. <a href=https://github.com/cortexproject/cortex/pull/4182>#4182</a></li><li>[BUGFIX] Ruler: Honor the evaluation delay for the <code>ALERTS</code> and <code>ALERTS_FOR_STATE</code> series. <a href=https://github.com/cortexproject/cortex/pull/4227>#4227</a></li><li>[BUGFIX] Make multiple Get requests instead of MGet on Redis Cluster. <a href=https://github.com/cortexproject/cortex/pull/4056>#4056</a></li><li>[BUGFIX] Ingester: fix issue where runtime limits erroneously override default limits. <a href=https://github.com/cortexproject/cortex/pull/4246>#4246</a></li><li>[BUGFIX] Ruler: fix startup in single-binary mode when the new <code>ruler_storage</code> is used. <a href=https://github.com/cortexproject/cortex/pull/4252>#4252</a></li><li>[BUGFIX] Querier: fix queries failing with &ldquo;at least 1 healthy replica required, could only find 0&rdquo; error right after scaling up store-gateways until they&rsquo;re ACTIVE in the ring. <a href=https://github.com/cortexproject/cortex/pull/4263>#4263</a></li><li>[BUGFIX] Store-gateway: when blocks sharding is enabled, do not load all blocks in each store-gateway in case of a cold startup, but load only blocks owned by the store-gateway replica. <a href=https://github.com/cortexproject/cortex/pull/4271>#4271</a></li><li>[BUGFIX] Memberlist: fix to setting the default configuration value for <code>-memberlist.retransmit-factor</code> when not provided. This should improve propagation delay of the ring state (including, but not limited to, tombstones). Note that if the configuration is already explicitly given, this fix has no effect. <a href=https://github.com/cortexproject/cortex/pull/4269>#4269</a></li><li>[BUGFIX] Querier: Fix issue where samples in a chunk might get skipped by batch iterator. <a href=https://github.com/cortexproject/cortex/pull/4218>#4218</a></li></ul><h2 id=blocksconvert>Blocksconvert</h2><ul><li>[ENHANCEMENT] Scanner: add support for DynamoDB (v9 schema only). <a href=https://github.com/cortexproject/cortex/pull/3828>#3828</a></li><li>[ENHANCEMENT] Add Cassandra support. <a href=https://github.com/cortexproject/cortex/pull/3795>#3795</a></li><li>[ENHANCEMENT] Scanner: retry failed uploads. <a href=https://github.com/cortexproject/cortex/pull/4188>#4188</a></li></ul><h2 id=190--2021-05-14>1.9.0 / 2021-05-14</h2><ul><li>[CHANGE] Alertmanager now removes local files after Alertmanager is no longer running for removed or resharded user. <a href=https://github.com/cortexproject/cortex/pull/3910>#3910</a></li><li>[CHANGE] Alertmanager now stores local files in per-tenant folders. Files stored by Alertmanager previously are migrated to new hierarchy. Support for this migration will be removed in Cortex 1.11. <a href=https://github.com/cortexproject/cortex/pull/3910>#3910</a></li><li>[CHANGE] Ruler: deprecated <code>-ruler.storage.*</code> CLI flags (and their respective YAML config options) in favour of <code>-ruler-storage.*</code>. The deprecated config will be removed in Cortex 1.11. <a href=https://github.com/cortexproject/cortex/pull/3945>#3945</a></li><li>[CHANGE] Alertmanager: deprecated <code>-alertmanager.storage.*</code> CLI flags (and their respective YAML config options) in favour of <code>-alertmanager-storage.*</code>. This change doesn&rsquo;t apply to <code>alertmanager.storage.path</code> and <code>alertmanager.storage.retention</code>. The deprecated config will be removed in Cortex 1.11. <a href=https://github.com/cortexproject/cortex/pull/4002>#4002</a></li><li>[CHANGE] Alertmanager: removed <code>-cluster.</code> CLI flags deprecated in Cortex 1.7. The new config options to use are: <a href=https://github.com/cortexproject/cortex/pull/3946>#3946</a><ul><li><code>-alertmanager.cluster.listen-address</code> instead of <code>-cluster.listen-address</code></li><li><code>-alertmanager.cluster.advertise-address</code> instead of <code>-cluster.advertise-address</code></li><li><code>-alertmanager.cluster.peers</code> instead of <code>-cluster.peer</code></li><li><code>-alertmanager.cluster.peer-timeout</code> instead of <code>-cluster.peer-timeout</code></li></ul></li><li>[CHANGE] Blocks storage: removed the config option <code>-blocks-storage.bucket-store.index-cache.postings-compression-enabled</code>, which was deprecated in Cortex 1.6. Postings compression is always enabled. <a href=https://github.com/cortexproject/cortex/pull/4101>#4101</a></li><li>[CHANGE] Querier: removed the config option <code>-store.max-look-back-period</code>, which was deprecated in Cortex 1.6 and was used only by the chunks storage. You should use <code>-querier.max-query-lookback</code> instead. <a href=https://github.com/cortexproject/cortex/pull/4101>#4101</a></li><li>[CHANGE] Query Frontend: removed the config option <code>-querier.compress-http-responses</code>, which was deprecated in Cortex 1.6. You should use<code>-api.response-compression-enabled</code> instead. <a href=https://github.com/cortexproject/cortex/pull/4101>#4101</a></li><li>[CHANGE] Runtime-config / overrides: removed the config options <code>-limits.per-user-override-config</code> (use <code>-runtime-config.file</code>) and <code>-limits.per-user-override-period</code> (use <code>-runtime-config.reload-period</code>), both deprecated since Cortex 0.6.0. <a href=https://github.com/cortexproject/cortex/pull/4112>#4112</a></li><li>[CHANGE] Cortex now fails fast on startup if unable to connect to the ring backend. <a href=https://github.com/cortexproject/cortex/pull/4068>#4068</a></li><li>[FEATURE] The following features have been marked as stable: <a href=https://github.com/cortexproject/cortex/pull/4101>#4101</a><ul><li>Shuffle-sharding</li><li>Querier support for querying chunks and blocks store at the same time</li><li>Tracking of active series and exporting them as metrics (<code>-ingester.active-series-metrics-enabled</code> and related flags)</li><li>Blocks storage: lazy mmap of block indexes in the store-gateway (<code>-blocks-storage.bucket-store.index-header-lazy-loading-enabled</code>)</li><li>Ingester: close idle TSDB and remove them from local disk (<code>-blocks-storage.tsdb.close-idle-tsdb-timeout</code>)</li></ul></li><li>[FEATURE] Memberlist: add TLS configuration options for the memberlist transport layer used by the gossip KV store. <a href=https://github.com/cortexproject/cortex/pull/4046>#4046</a><ul><li>New flags added for memberlist communication:<ul><li><code>-memberlist.tls-enabled</code></li><li><code>-memberlist.tls-cert-path</code></li><li><code>-memberlist.tls-key-path</code></li><li><code>-memberlist.tls-ca-path</code></li><li><code>-memberlist.tls-server-name</code></li><li><code>-memberlist.tls-insecure-skip-verify</code></li></ul></li></ul></li><li>[FEATURE] Ruler: added <code>local</code> backend support to the ruler storage configuration under the <code>-ruler-storage.</code> flag prefix. <a href=https://github.com/cortexproject/cortex/pull/3932>#3932</a></li><li>[ENHANCEMENT] Upgraded Docker base images to <code>alpine:3.13</code>. <a href=https://github.com/cortexproject/cortex/pull/4042>#4042</a></li><li>[ENHANCEMENT] Blocks storage: reduce ingester memory by eliminating series reference cache. <a href=https://github.com/cortexproject/cortex/pull/3951>#3951</a></li><li>[ENHANCEMENT] Ruler: optimized <code>&lt;prefix>/api/v1/rules</code> and <code>&lt;prefix>/api/v1/alerts</code> when ruler sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/3916>#3916</a></li><li>[ENHANCEMENT] Ruler: added the following metrics when ruler sharding is enabled: <a href=https://github.com/cortexproject/cortex/pull/3916>#3916</a><ul><li><code>cortex_ruler_clients</code></li><li><code>cortex_ruler_client_request_duration_seconds</code></li></ul></li><li>[ENHANCEMENT] Alertmanager: Add API endpoint to list all tenant alertmanager configs: <code>GET /multitenant_alertmanager/configs</code>. <a href=https://github.com/cortexproject/cortex/pull/3529>#3529</a></li><li>[ENHANCEMENT] Ruler: Add API endpoint to list all tenant ruler rule groups: <code>GET /ruler/rule_groups</code>. <a href=https://github.com/cortexproject/cortex/pull/3529>#3529</a></li><li>[ENHANCEMENT] Query-frontend/scheduler: added querier forget delay (<code>-query-frontend.querier-forget-delay</code> and <code>-query-scheduler.querier-forget-delay</code>) to mitigate the blast radius in the event queriers crash because of a repeatedly sent &ldquo;query of death&rdquo; when shuffle-sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/3901>#3901</a></li><li>[ENHANCEMENT] Query-frontend: reduced memory allocations when serializing query response. <a href=https://github.com/cortexproject/cortex/pull/3964>#3964</a></li><li>[ENHANCEMENT] Querier / ruler: some optimizations to PromQL query engine. <a href=https://github.com/cortexproject/cortex/pull/3934>#3934</a> <a href=https://github.com/cortexproject/cortex/pull/3989>#3989</a></li><li>[ENHANCEMENT] Ingester: reduce CPU and memory when an high number of errors are returned by the ingester on the write path with the blocks storage. <a href=https://github.com/cortexproject/cortex/pull/3969>#3969</a> <a href=https://github.com/cortexproject/cortex/pull/3971>#3971</a> <a href=https://github.com/cortexproject/cortex/pull/3973>#3973</a></li><li>[ENHANCEMENT] Distributor: reduce CPU and memory when an high number of errors are returned by the distributor on the write path. <a href=https://github.com/cortexproject/cortex/pull/3990>#3990</a></li><li>[ENHANCEMENT] Put metric before label value in the &ldquo;label value too long&rdquo; error message. <a href=https://github.com/cortexproject/cortex/pull/4018>#4018</a></li><li>[ENHANCEMENT] Allow use of <code>y|w|d</code> suffixes for duration related limits and per-tenant limits. <a href=https://github.com/cortexproject/cortex/pull/4044>#4044</a></li><li>[ENHANCEMENT] Query-frontend: Small optimization on top of PR <a href=https://github.com/cortexproject/cortex/pull/3968>#3968</a> to avoid unnecessary Extents merging. <a href=https://github.com/cortexproject/cortex/pull/4026>#4026</a></li><li>[ENHANCEMENT] Add a metric <code>cortex_compactor_compaction_interval_seconds</code> for the compaction interval config value. <a href=https://github.com/cortexproject/cortex/pull/4040>#4040</a></li><li>[ENHANCEMENT] Ingester: added following per-ingester (instance) experimental limits: max number of series in memory (<code>-ingester.instance-limits.max-series</code>), max number of users in memory (<code>-ingester.instance-limits.max-tenants</code>), max ingestion rate (<code>-ingester.instance-limits.max-ingestion-rate</code>), and max inflight requests (<code>-ingester.instance-limits.max-inflight-push-requests</code>). These limits are only used when using blocks storage. Limits can also be configured using runtime-config feature, and current values are exported as <code>cortex_ingester_instance_limits</code> metric. <a href=https://github.com/cortexproject/cortex/pull/3992>#3992</a>.</li><li>[ENHANCEMENT] Cortex is now built with Go 1.16. <a href=https://github.com/cortexproject/cortex/pull/4062>#4062</a></li><li>[ENHANCEMENT] Distributor: added per-distributor experimental limits: max number of inflight requests (<code>-distributor.instance-limits.max-inflight-push-requests</code>) and max ingestion rate in samples/sec (<code>-distributor.instance-limits.max-ingestion-rate</code>). If not set, these two are unlimited. Also added metrics to expose current values (<code>cortex_distributor_inflight_push_requests</code>, <code>cortex_distributor_ingestion_rate_samples_per_second</code>) as well as limits (<code>cortex_distributor_instance_limits</code> with various <code>limit</code> label values). <a href=https://github.com/cortexproject/cortex/pull/4071>#4071</a></li><li>[ENHANCEMENT] Ruler: Added <code>-ruler.enabled-tenants</code> and <code>-ruler.disabled-tenants</code> to explicitly enable or disable rules processing for specific tenants. <a href=https://github.com/cortexproject/cortex/pull/4074>#4074</a></li><li>[ENHANCEMENT] Block Storage Ingester: <code>/flush</code> now accepts two new parameters: <code>tenant</code> to specify tenant to flush and <code>wait=true</code> to make call synchronous. Multiple tenants can be specified by repeating <code>tenant</code> parameter. If no <code>tenant</code> is specified, all tenants are flushed, as before. <a href=https://github.com/cortexproject/cortex/pull/4073>#4073</a></li><li>[ENHANCEMENT] Alertmanager: validate configured <code>-alertmanager.web.external-url</code> and fail if ends with <code>/</code>. <a href=https://github.com/cortexproject/cortex/pull/4081>#4081</a></li><li>[ENHANCEMENT] Alertmanager: added <code>-alertmanager.receivers-firewall.block.cidr-networks</code> and <code>-alertmanager.receivers-firewall.block.private-addresses</code> to block specific network addresses in HTTP-based Alertmanager receiver integrations. <a href=https://github.com/cortexproject/cortex/pull/4085>#4085</a></li><li>[ENHANCEMENT] Allow configuration of Cassandra&rsquo;s host selection policy. <a href=https://github.com/cortexproject/cortex/pull/4069>#4069</a></li><li>[ENHANCEMENT] Store-gateway: retry syncing blocks if a per-tenant sync fails. <a href=https://github.com/cortexproject/cortex/pull/3975>#3975</a> <a href=https://github.com/cortexproject/cortex/pull/4088>#4088</a></li><li>[ENHANCEMENT] Add metric <code>cortex_tcp_connections</code> exposing the current number of accepted TCP connections. <a href=https://github.com/cortexproject/cortex/pull/4099>#4099</a></li><li>[ENHANCEMENT] Querier: Allow federated queries to run concurrently. <a href=https://github.com/cortexproject/cortex/pull/4065>#4065</a></li><li>[ENHANCEMENT] Label Values API call now supports <code>match[]</code> parameter when querying blocks on storage (assuming <code>-querier.query-store-for-labels-enabled</code> is enabled). <a href=https://github.com/cortexproject/cortex/pull/4133>#4133</a></li><li>[BUGFIX] Ruler-API: fix bug where <code>/api/v1/rules/&lt;namespace>/&lt;group_name></code> endpoint return <code>400</code> instead of <code>404</code>. <a href=https://github.com/cortexproject/cortex/pull/4013>#4013</a></li><li>[BUGFIX] Distributor: reverted changes done to rate limiting in <a href=https://github.com/cortexproject/cortex/pull/3825>#3825</a>. <a href=https://github.com/cortexproject/cortex/pull/3948>#3948</a></li><li>[BUGFIX] Ingester: Fix race condition when opening and closing tsdb concurrently. <a href=https://github.com/cortexproject/cortex/pull/3959>#3959</a></li><li>[BUGFIX] Querier: streamline tracing spans. <a href=https://github.com/cortexproject/cortex/pull/3924>#3924</a></li><li>[BUGFIX] Ruler Storage: ignore objects with empty namespace or group in the name. <a href=https://github.com/cortexproject/cortex/pull/3999>#3999</a></li><li>[BUGFIX] Distributor: fix issue causing distributors to not extend the replication set because of failing instances when zone-aware replication is enabled. <a href=https://github.com/cortexproject/cortex/pull/3977>#3977</a></li><li>[BUGFIX] Query-frontend: Fix issue where cached entry size keeps increasing when making tiny query repeatedly. <a href=https://github.com/cortexproject/cortex/pull/3968>#3968</a></li><li>[BUGFIX] Compactor: <code>-compactor.blocks-retention-period</code> now supports weeks (<code>w</code>) and years (<code>y</code>). <a href=https://github.com/cortexproject/cortex/pull/4027>#4027</a></li><li>[BUGFIX] Querier: returning 422 (instead of 500) when query hits <code>max_chunks_per_query</code> limit with block storage, when the limit is hit in the store-gateway. <a href=https://github.com/cortexproject/cortex/pull/3937>#3937</a></li><li>[BUGFIX] Ruler: Rule group limit enforcement should now allow the same number of rules in a group as the limit. <a href=https://github.com/cortexproject/cortex/pull/3616>#3616</a></li><li>[BUGFIX] Frontend, Query-scheduler: allow querier to notify about shutdown without providing any authentication. <a href=https://github.com/cortexproject/cortex/pull/4066>#4066</a></li><li>[BUGFIX] Querier: fixed race condition causing queries to fail right after querier startup with the &ldquo;empty ring&rdquo; error. <a href=https://github.com/cortexproject/cortex/pull/4068>#4068</a></li><li>[BUGFIX] Compactor: Increment <code>cortex_compactor_runs_failed_total</code> if compactor failed compact a single tenant. <a href=https://github.com/cortexproject/cortex/pull/4094>#4094</a></li><li>[BUGFIX] Tracing: hot fix to avoid the Jaeger tracing client to indefinitely block the Cortex process shutdown in case the HTTP connection to the tracing backend is blocked. <a href=https://github.com/cortexproject/cortex/pull/4134>#4134</a></li><li>[BUGFIX] Forward proper EndsAt from ruler to Alertmanager inline with Prometheus behaviour. <a href=https://github.com/cortexproject/cortex/pull/4017>#4017</a></li><li>[BUGFIX] Querier: support filtering LabelValues with matchers when using tenant federation. <a href=https://github.com/cortexproject/cortex/pull/4277>#4277</a></li></ul><h2 id=blocksconvert-1>Blocksconvert</h2><ul><li>[ENHANCEMENT] Builder: add <code>-builder.timestamp-tolerance</code> option which may reduce block size by rounding timestamps to make difference whole seconds. <a href=https://github.com/cortexproject/cortex/pull/3891>#3891</a></li></ul><h2 id=181--2021-04-27>1.8.1 / 2021-04-27</h2><ul><li>[CHANGE] Fix for CVE-2021-31232: Local file disclosure vulnerability when <code>-experimental.alertmanager.enable-api</code> is used. The HTTP basic auth <code>password_file</code> can be used as an attack vector to send any file content via a webhook. The alertmanager templates can be used as an attack vector to send any file content because the alertmanager can load any text file specified in the templates list.</li></ul><h2 id=180--2021-03-24>1.8.0 / 2021-03-24</h2><ul><li>[CHANGE] Alertmanager: Don&rsquo;t expose cluster information to tenants via the <code>/alertmanager/api/v1/status</code> API endpoint when operating with clustering enabled. <a href=https://github.com/cortexproject/cortex/pull/3903>#3903</a></li><li>[CHANGE] Ingester: don&rsquo;t update internal &ldquo;last updated&rdquo; timestamp of TSDB if tenant only sends invalid samples. This affects how &ldquo;idle&rdquo; time is computed. <a href=https://github.com/cortexproject/cortex/pull/3727>#3727</a></li><li>[CHANGE] Require explicit flag <code>-&lt;prefix>.tls-enabled</code> to enable TLS in GRPC clients. Previously it was enough to specify a TLS flag to enable TLS validation. <a href=https://github.com/cortexproject/cortex/pull/3156>#3156</a></li><li>[CHANGE] Query-frontend: removed <code>-querier.split-queries-by-day</code> (deprecated in Cortex 0.4.0). Please use <code>-querier.split-queries-by-interval</code> instead. <a href=https://github.com/cortexproject/cortex/pull/3813>#3813</a></li><li>[CHANGE] Store-gateway: the chunks pool controlled by <code>-blocks-storage.bucket-store.max-chunk-pool-bytes</code> is now shared across all tenants. <a href=https://github.com/cortexproject/cortex/pull/3830>#3830</a></li><li>[CHANGE] Ingester: return error code 400 instead of 429 when per-user/per-tenant series/metadata limits are reached. <a href=https://github.com/cortexproject/cortex/pull/3833>#3833</a></li><li>[CHANGE] Compactor: add <code>reason</code> label to <code>cortex_compactor_blocks_marked_for_deletion_total</code> metric. Source blocks marked for deletion by compactor are labelled as <code>compaction</code>, while blocks passing the retention period are labelled as <code>retention</code>. <a href=https://github.com/cortexproject/cortex/pull/3879>#3879</a></li><li>[CHANGE] Alertmanager: the <code>DELETE /api/v1/alerts</code> is now idempotent. No error is returned if the alertmanager config doesn&rsquo;t exist. <a href=https://github.com/cortexproject/cortex/pull/3888>#3888</a></li><li>[FEATURE] Experimental Ruler Storage: Add a separate set of configuration options to configure the ruler storage backend under the <code>-ruler-storage.</code> flag prefix. All blocks storage bucket clients and the config service are currently supported. Clients using this implementation will only be enabled if the existing <code>-ruler.storage</code> flags are left unset. <a href=https://github.com/cortexproject/cortex/pull/3805>#3805</a> <a href=https://github.com/cortexproject/cortex/pull/3864>#3864</a></li><li>[FEATURE] Experimental Alertmanager Storage: Add a separate set of configuration options to configure the alertmanager storage backend under the <code>-alertmanager-storage.</code> flag prefix. All blocks storage bucket clients and the config service are currently supported. Clients using this implementation will only be enabled if the existing <code>-alertmanager.storage</code> flags are left unset. <a href=https://github.com/cortexproject/cortex/pull/3888>#3888</a></li><li>[FEATURE] Adds support to S3 server-side encryption using KMS. The S3 server-side encryption config can be overridden on a per-tenant basis for the blocks storage, ruler and alertmanager. Deprecated <code>-&lt;prefix>.s3.sse-encryption</code>, please use the following CLI flags that have been added. <a href=https://github.com/cortexproject/cortex/pull/3651>#3651</a> <a href=https://github.com/cortexproject/cortex/pull/3810>#3810</a> <a href=https://github.com/cortexproject/cortex/pull/3811>#3811</a> <a href=https://github.com/cortexproject/cortex/pull/3870>#3870</a> <a href=https://github.com/cortexproject/cortex/pull/3886>#3886</a> <a href=https://github.com/cortexproject/cortex/pull/3906>#3906</a><ul><li><code>-&lt;prefix>.s3.sse.type</code></li><li><code>-&lt;prefix>.s3.sse.kms-key-id</code></li><li><code>-&lt;prefix>.s3.sse.kms-encryption-context</code></li></ul></li><li>[FEATURE] Querier: Enable <code>@ &lt;timestamp></code> modifier in PromQL using the new <code>-querier.at-modifier-enabled</code> flag. <a href=https://github.com/cortexproject/cortex/pull/3744>#3744</a></li><li>[FEATURE] Overrides Exporter: Add <code>overrides-exporter</code> module for exposing per-tenant resource limit overrides as metrics. It is not included in <code>all</code> target (single-binary mode), and must be explicitly enabled. <a href=https://github.com/cortexproject/cortex/pull/3785>#3785</a></li><li>[FEATURE] Experimental thanosconvert: introduce an experimental tool <code>thanosconvert</code> to migrate Thanos block metadata to Cortex metadata. <a href=https://github.com/cortexproject/cortex/pull/3770>#3770</a></li><li>[FEATURE] Alertmanager: It now shards the <code>/api/v1/alerts</code> API using the ring when sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/3671>#3671</a><ul><li>Added <code>-alertmanager.max-recv-msg-size</code> (defaults to 16M) to limit the size of HTTP request body handled by the alertmanager.</li><li>New flags added for communication between alertmanagers:<ul><li><code>-alertmanager.max-recv-msg-size</code></li><li><code>-alertmanager.alertmanager-client.remote-timeout</code></li><li><code>-alertmanager.alertmanager-client.tls-enabled</code></li><li><code>-alertmanager.alertmanager-client.tls-cert-path</code></li><li><code>-alertmanager.alertmanager-client.tls-key-path</code></li><li><code>-alertmanager.alertmanager-client.tls-ca-path</code></li><li><code>-alertmanager.alertmanager-client.tls-server-name</code></li><li><code>-alertmanager.alertmanager-client.tls-insecure-skip-verify</code></li></ul></li></ul></li><li>[FEATURE] Compactor: added blocks storage per-tenant retention support. This is configured via <code>-compactor.retention-period</code>, and can be overridden on a per-tenant basis. <a href=https://github.com/cortexproject/cortex/pull/3879>#3879</a></li><li>[ENHANCEMENT] Queries: Instrument queries that were discarded due to the configured <code>max_outstanding_requests_per_tenant</code>. <a href=https://github.com/cortexproject/cortex/pull/3894>#3894</a><ul><li><code>cortex_query_frontend_discarded_requests_total</code></li><li><code>cortex_query_scheduler_discarded_requests_total</code></li></ul></li><li>[ENHANCEMENT] Ruler: Add TLS and explicit basis authentication configuration options for the HTTP client the ruler uses to communicate with the alertmanager. <a href=https://github.com/cortexproject/cortex/pull/3752>#3752</a><ul><li><code>-ruler.alertmanager-client.basic-auth-username</code>: Configure the basic authentication username used by the client. Takes precedent over a URL configured username.</li><li><code>-ruler.alertmanager-client.basic-auth-password</code>: Configure the basic authentication password used by the client. Takes precedent over a URL configured password.</li><li><code>-ruler.alertmanager-client.tls-ca-path</code>: File path to the CA file.</li><li><code>-ruler.alertmanager-client.tls-cert-path</code>: File path to the TLS certificate.</li><li><code>-ruler.alertmanager-client.tls-insecure-skip-verify</code>: Boolean to disable verifying the certificate.</li><li><code>-ruler.alertmanager-client.tls-key-path</code>: File path to the TLS key certificate.</li><li><code>-ruler.alertmanager-client.tls-server-name</code>: Expected name on the TLS certificate.</li></ul></li><li>[ENHANCEMENT] Ingester: exposed metric <code>cortex_ingester_oldest_unshipped_block_timestamp_seconds</code>, tracking the unix timestamp of the oldest TSDB block not shipped to the storage yet. <a href=https://github.com/cortexproject/cortex/pull/3705>#3705</a></li><li>[ENHANCEMENT] Prometheus upgraded. <a href=https://github.com/cortexproject/cortex/pull/3739>#3739</a> <a href=https://github.com/cortexproject/cortex/pull/3806>#3806</a><ul><li>Avoid unnecessary <code>runtime.GC()</code> during compactions.</li><li>Prevent compaction loop in TSDB on data gap.</li></ul></li><li>[ENHANCEMENT] Query-Frontend now returns server side performance metrics using <code>Server-Timing</code> header when query stats is enabled. <a href=https://github.com/cortexproject/cortex/pull/3685>#3685</a></li><li>[ENHANCEMENT] Runtime Config: Add a <code>mode</code> query parameter for the runtime config endpoint. <code>/runtime_config?mode=diff</code> now shows the YAML runtime configuration with all values that differ from the defaults. <a href=https://github.com/cortexproject/cortex/pull/3700>#3700</a></li><li>[ENHANCEMENT] Distributor: Enable downstream projects to wrap distributor push function and access the deserialized write requests before/after they are pushed. <a href=https://github.com/cortexproject/cortex/pull/3755>#3755</a></li><li>[ENHANCEMENT] Add flag <code>-&lt;prefix>.tls-server-name</code> to require a specific server name instead of the hostname on the certificate. <a href=https://github.com/cortexproject/cortex/pull/3156>#3156</a></li><li>[ENHANCEMENT] Alertmanager: Remove a tenant&rsquo;s alertmanager instead of pausing it as we determine it is no longer needed. <a href=https://github.com/cortexproject/cortex/pull/3722>#3722</a></li><li>[ENHANCEMENT] Blocks storage: added more configuration options to S3 client. <a href=https://github.com/cortexproject/cortex/pull/3775>#3775</a><ul><li><code>-blocks-storage.s3.tls-handshake-timeout</code>: Maximum time to wait for a TLS handshake. 0 means no limit.</li><li><code>-blocks-storage.s3.expect-continue-timeout</code>: The time to wait for a server&rsquo;s first response headers after fully writing the request headers if the request has an Expect header. 0 to send the request body immediately.</li><li><code>-blocks-storage.s3.max-idle-connections</code>: Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit.</li><li><code>-blocks-storage.s3.max-idle-connections-per-host</code>: Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used.</li><li><code>-blocks-storage.s3.max-connections-per-host</code>: Maximum number of connections per host. 0 means no limit.</li></ul></li><li>[ENHANCEMENT] Ingester: when tenant&rsquo;s TSDB is closed, Ingester now removes pushed metrics-metadata from memory, and removes metadata (<code>cortex_ingester_memory_metadata</code>, <code>cortex_ingester_memory_metadata_created_total</code>, <code>cortex_ingester_memory_metadata_removed_total</code>) and validation metrics (<code>cortex_discarded_samples_total</code>, <code>cortex_discarded_metadata_total</code>). <a href=https://github.com/cortexproject/cortex/pull/3782>#3782</a></li><li>[ENHANCEMENT] Distributor: cleanup metrics for inactive tenants. <a href=https://github.com/cortexproject/cortex/pull/3784>#3784</a></li><li>[ENHANCEMENT] Ingester: Have ingester to re-emit following TSDB metrics. <a href=https://github.com/cortexproject/cortex/pull/3800>#3800</a><ul><li><code>cortex_ingester_tsdb_blocks_loaded</code></li><li><code>cortex_ingester_tsdb_reloads_total</code></li><li><code>cortex_ingester_tsdb_reloads_failures_total</code></li><li><code>cortex_ingester_tsdb_symbol_table_size_bytes</code></li><li><code>cortex_ingester_tsdb_storage_blocks_bytes</code></li><li><code>cortex_ingester_tsdb_time_retentions_total</code></li></ul></li><li>[ENHANCEMENT] Querier: distribute workload across <code>-store-gateway.sharding-ring.replication-factor</code> store-gateway replicas when querying blocks and <code>-store-gateway.sharding-enabled=true</code>. <a href=https://github.com/cortexproject/cortex/pull/3824>#3824</a></li><li>[ENHANCEMENT] Distributor / HA Tracker: added cleanup of unused elected HA replicas from KV store. Added following metrics to monitor this process: <a href=https://github.com/cortexproject/cortex/pull/3809>#3809</a><ul><li><code>cortex_ha_tracker_replicas_cleanup_started_total</code></li><li><code>cortex_ha_tracker_replicas_cleanup_marked_for_deletion_total</code></li><li><code>cortex_ha_tracker_replicas_cleanup_deleted_total</code></li><li><code>cortex_ha_tracker_replicas_cleanup_delete_failed_total</code></li></ul></li><li>[ENHANCEMENT] Ruler now has new API endpoint <code>/ruler/delete_tenant_config</code> that can be used to delete all ruler groups for tenant. It is intended to be used by administrators who wish to clean up state after removed user. Note that this endpoint is enabled regardless of <code>-experimental.ruler.enable-api</code>. <a href=https://github.com/cortexproject/cortex/pull/3750>#3750</a> <a href=https://github.com/cortexproject/cortex/pull/3899>#3899</a></li><li>[ENHANCEMENT] Query-frontend, query-scheduler: cleanup metrics for inactive tenants. <a href=https://github.com/cortexproject/cortex/pull/3826>#3826</a></li><li>[ENHANCEMENT] Blocks storage: added <code>-blocks-storage.s3.region</code> support to S3 client configuration. <a href=https://github.com/cortexproject/cortex/pull/3811>#3811</a></li><li>[ENHANCEMENT] Distributor: Remove cached subrings for inactive users when using shuffle sharding. <a href=https://github.com/cortexproject/cortex/pull/3849>#3849</a></li><li>[ENHANCEMENT] Store-gateway: Reduced memory used to fetch chunks at query time. <a href=https://github.com/cortexproject/cortex/pull/3855>#3855</a></li><li>[ENHANCEMENT] Ingester: attempt to prevent idle compaction from happening in concurrent ingesters by introducing a 25% jitter to the configured idle timeout (<code>-blocks-storage.tsdb.head-compaction-idle-timeout</code>). <a href=https://github.com/cortexproject/cortex/pull/3850>#3850</a></li><li>[ENHANCEMENT] Compactor: cleanup local files for users that are no longer owned by compactor. <a href=https://github.com/cortexproject/cortex/pull/3851>#3851</a></li><li>[ENHANCEMENT] Store-gateway: close empty bucket stores, and delete leftover local files for tenants that no longer belong to store-gateway. <a href=https://github.com/cortexproject/cortex/pull/3853>#3853</a></li><li>[ENHANCEMENT] Store-gateway: added metrics to track partitioner behaviour. <a href=https://github.com/cortexproject/cortex/pull/3877>#3877</a><ul><li><code>cortex_bucket_store_partitioner_requested_bytes_total</code></li><li><code>cortex_bucket_store_partitioner_requested_ranges_total</code></li><li><code>cortex_bucket_store_partitioner_expanded_bytes_total</code></li><li><code>cortex_bucket_store_partitioner_expanded_ranges_total</code></li></ul></li><li>[ENHANCEMENT] Store-gateway: added metrics to monitor chunk buffer pool behaviour. <a href=https://github.com/cortexproject/cortex/pull/3880>#3880</a><ul><li><code>cortex_bucket_store_chunk_pool_requested_bytes_total</code></li><li><code>cortex_bucket_store_chunk_pool_returned_bytes_total</code></li></ul></li><li>[ENHANCEMENT] Alertmanager: load alertmanager configurations from object storage concurrently, and only load necessary configurations, speeding configuration synchronization process and executing fewer &ldquo;GET object&rdquo; operations to the storage when sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/3898>#3898</a></li><li>[ENHANCEMENT] Ingester (blocks storage): Ingester can now stream entire chunks instead of individual samples to the querier. At the moment this feature must be explicitly enabled either by using <code>-ingester.stream-chunks-when-using-blocks</code> flag or <code>ingester_stream_chunks_when_using_blocks</code> (boolean) field in runtime config file, but these configuration options are temporary and will be removed when feature is stable. <a href=https://github.com/cortexproject/cortex/pull/3889>#3889</a></li><li>[ENHANCEMENT] Alertmanager: New endpoint <code>/multitenant_alertmanager/delete_tenant_config</code> to delete configuration for tenant identified by <code>X-Scope-OrgID</code> header. This is an internal endpoint, available even if Alertmanager API is not enabled by using <code>-experimental.alertmanager.enable-api</code>. <a href=https://github.com/cortexproject/cortex/pull/3900>#3900</a></li><li>[ENHANCEMENT] MemCached: Add <code>max_item_size</code> support. <a href=https://github.com/cortexproject/cortex/pull/3929>#3929</a></li><li>[BUGFIX] Cortex: Fixed issue where fatal errors and various log messages where not logged. <a href=https://github.com/cortexproject/cortex/pull/3778>#3778</a></li><li>[BUGFIX] HA Tracker: don&rsquo;t track as error in the <code>cortex_kv_request_duration_seconds</code> metric a CAS operation intentionally aborted. <a href=https://github.com/cortexproject/cortex/pull/3745>#3745</a></li><li>[BUGFIX] Querier / ruler: do not log &ldquo;error removing stale clients&rdquo; if the ring is empty. <a href=https://github.com/cortexproject/cortex/pull/3761>#3761</a></li><li>[BUGFIX] Store-gateway: fixed a panic caused by a race condition when the index-header lazy loading is enabled. <a href=https://github.com/cortexproject/cortex/pull/3775>#3775</a> <a href=https://github.com/cortexproject/cortex/pull/3789>#3789</a></li><li>[BUGFIX] Compactor: fixed &ldquo;could not guess file size&rdquo; log when uploading blocks deletion marks to the global location. <a href=https://github.com/cortexproject/cortex/pull/3807>#3807</a></li><li>[BUGFIX] Prevent panic at start if the http_prefix setting doesn&rsquo;t have a valid value. <a href=https://github.com/cortexproject/cortex/pull/3796>#3796</a></li><li>[BUGFIX] Memberlist: fixed panic caused by race condition in <code>armon/go-metrics</code> used by memberlist client. <a href=https://github.com/cortexproject/cortex/pull/3725>#3725</a></li><li>[BUGFIX] Querier: returning 422 (instead of 500) when query hits <code>max_chunks_per_query</code> limit with block storage. <a href=https://github.com/cortexproject/cortex/pull/3895>#3895</a></li><li>[BUGFIX] Alertmanager: Ensure that experimental <code>/api/v1/alerts</code> endpoints work when <code>-http.prefix</code> is empty. <a href=https://github.com/cortexproject/cortex/pull/3905>#3905</a></li><li>[BUGFIX] Chunk store: fix panic in inverted index when deleted fingerprint is no longer in the index. <a href=https://github.com/cortexproject/cortex/pull/3543>#3543</a></li></ul><h2 id=171--2021-04-27>1.7.1 / 2021-04-27</h2><ul><li>[CHANGE] Fix for CVE-2021-31232: Local file disclosure vulnerability when <code>-experimental.alertmanager.enable-api</code> is used. The HTTP basic auth <code>password_file</code> can be used as an attack vector to send any file content via a webhook. The alertmanager templates can be used as an attack vector to send any file content because the alertmanager can load any text file specified in the templates list.</li></ul><h2 id=170--2021-02-23>1.7.0 / 2021-02-23</h2><p>Note the blocks storage compactor runs a migration task at startup in this version, which can take many minutes and use a lot of RAM.
 <a href=https://cortexmetrics.io/docs/blocks-storage/production-tips/#ensure-deletion-marks-migration-is-disabled-after-first-run>Turn this off after first run</a>.</p><ul><li>[CHANGE] FramedSnappy encoding support has been removed from Push and Remote Read APIs. This means Prometheus 1.6 support has been removed and the oldest Prometheus version supported in the remote write is 1.7. <a href=https://github.com/cortexproject/cortex/pull/3682>#3682</a></li><li>[CHANGE] Ruler: removed the flag <code>-ruler.evaluation-delay-duration-deprecated</code> which was deprecated in 1.4.0. Please use the <code>ruler_evaluation_delay_duration</code> per-tenant limit instead. <a href=https://github.com/cortexproject/cortex/pull/3694>#3694</a></li><li>[CHANGE] Removed the flags <code>-&lt;prefix>.grpc-use-gzip-compression</code> which were deprecated in 1.3.0: <a href=https://github.com/cortexproject/cortex/pull/3694>#3694</a><ul><li><code>-query-scheduler.grpc-client-config.grpc-use-gzip-compression</code>: use <code>-query-scheduler.grpc-client-config.grpc-compression</code> instead</li><li><code>-frontend.grpc-client-config.grpc-use-gzip-compression</code>: use <code>-frontend.grpc-client-config.grpc-compression</code> instead</li><li><code>-ruler.client.grpc-use-gzip-compression</code>: use <code>-ruler.client.grpc-compression</code> instead</li><li><code>-bigtable.grpc-use-gzip-compression</code>: use <code>-bigtable.grpc-compression</code> instead</li><li><code>-ingester.client.grpc-use-gzip-compression</code>: use <code>-ingester.client.grpc-compression</code> instead</li><li><code>-querier.frontend-client.grpc-use-gzip-compression</code>: use <code>-querier.frontend-client.grpc-compression</code> instead</li></ul></li><li>[CHANGE] Querier: it&rsquo;s not required to set <code>-frontend.query-stats-enabled=true</code> in the querier anymore to enable query statistics logging in the query-frontend. The flag is now required to be configured only in the query-frontend and it will be propagated to the queriers. <a href=https://github.com/cortexproject/cortex/pull/3595>#3595</a> <a href=https://github.com/cortexproject/cortex/pull/3695>#3695</a></li><li>[CHANGE] Blocks storage: compactor is now required when running a Cortex cluster with the blocks storage, because it also keeps the bucket index updated. <a href=https://github.com/cortexproject/cortex/pull/3583>#3583</a></li><li>[CHANGE] Blocks storage: block deletion marks are now stored in a per-tenant global markers/ location too, other than within the block location. The compactor, at startup, will copy deletion marks from the block location to the global location. This migration is required only once, so it can be safely disabled via <code>-compactor.block-deletion-marks-migration-enabled=false</code> after new compactor has successfully started at least once in the cluster. <a href=https://github.com/cortexproject/cortex/pull/3583>#3583</a></li><li>[CHANGE] OpenStack Swift: the default value for the <code>-ruler.storage.swift.container-name</code> and <code>-swift.container-name</code> config options has changed from <code>cortex</code> to empty string. If you were relying on the default value, please set it back to <code>cortex</code>. <a href=https://github.com/cortexproject/cortex/pull/3660>#3660</a></li><li>[CHANGE] HA Tracker: configured replica label is now verified against label value length limit (<code>-validation.max-length-label-value</code>). <a href=https://github.com/cortexproject/cortex/pull/3668>#3668</a></li><li>[CHANGE] Distributor: <code>extend_writes</code> field in YAML configuration has moved from <code>lifecycler</code> (inside <code>ingester_config</code>) to <code>distributor_config</code>. This doesn&rsquo;t affect command line option <code>-distributor.extend-writes</code>, which stays the same. <a href=https://github.com/cortexproject/cortex/pull/3719>#3719</a></li><li>[CHANGE] Alertmanager: Deprecated <code>-cluster.</code> CLI flags in favor of their <code>-alertmanager.cluster.</code> equivalent. The deprecated flags (and their respective YAML config options) are: <a href=https://github.com/cortexproject/cortex/pull/3677>#3677</a><ul><li><code>-cluster.listen-address</code> in favor of <code>-alertmanager.cluster.listen-address</code></li><li><code>-cluster.advertise-address</code> in favor of <code>-alertmanager.cluster.advertise-address</code></li><li><code>-cluster.peer</code> in favor of <code>-alertmanager.cluster.peers</code></li><li><code>-cluster.peer-timeout</code> in favor of <code>-alertmanager.cluster.peer-timeout</code></li></ul></li><li>[CHANGE] Blocks storage: the default value of <code>-blocks-storage.bucket-store.sync-interval</code> has been changed from <code>5m</code> to <code>15m</code>. <a href=https://github.com/cortexproject/cortex/pull/3724>#3724</a></li><li>[FEATURE] Querier: Queries can be federated across multiple tenants. The tenants IDs involved need to be specified separated by a <code>|</code> character in the <code>X-Scope-OrgID</code> request header. This is an experimental feature, which can be enabled by setting <code>-tenant-federation.enabled=true</code> on all Cortex services. <a href=https://github.com/cortexproject/cortex/pull/3250>#3250</a></li><li>[FEATURE] Alertmanager: introduced the experimental option <code>-alertmanager.sharding-enabled</code> to shard tenants across multiple Alertmanager instances. This feature is still under heavy development and its usage is discouraged. The following new metrics are exported by the Alertmanager: <a href=https://github.com/cortexproject/cortex/pull/3664>#3664</a><ul><li><code>cortex_alertmanager_ring_check_errors_total</code></li><li><code>cortex_alertmanager_sync_configs_total</code></li><li><code>cortex_alertmanager_sync_configs_failed_total</code></li><li><code>cortex_alertmanager_tenants_discovered</code></li><li><code>cortex_alertmanager_tenants_owned</code></li></ul></li><li>[ENHANCEMENT] Allow specifying JAEGER_ENDPOINT instead of sampling server or local agent port. <a href=https://github.com/cortexproject/cortex/pull/3682>#3682</a></li><li>[ENHANCEMENT] Blocks storage: introduced a per-tenant bucket index, periodically updated by the compactor, used to avoid full bucket scanning done by queriers, store-gateways and rulers. The bucket index is updated by the compactor during blocks cleanup, on every <code>-compactor.cleanup-interval</code>. <a href=https://github.com/cortexproject/cortex/pull/3553>#3553</a> <a href=https://github.com/cortexproject/cortex/pull/3555>#3555</a> <a href=https://github.com/cortexproject/cortex/pull/3561>#3561</a> <a href=https://github.com/cortexproject/cortex/pull/3583>#3583</a> <a href=https://github.com/cortexproject/cortex/pull/3625>#3625</a> <a href=https://github.com/cortexproject/cortex/pull/3711>#3711</a> <a href=https://github.com/cortexproject/cortex/pull/3715>#3715</a></li><li>[ENHANCEMENT] Blocks storage: introduced an option <code>-blocks-storage.bucket-store.bucket-index.enabled</code> to enable the usage of the bucket index in the querier, store-gateway and ruler. When enabled, the querier, store-gateway and ruler will use the bucket index to find a tenant&rsquo;s blocks instead of running the periodic bucket scan. The following new metrics are exported by the querier and ruler: <a href=https://github.com/cortexproject/cortex/pull/3614>#3614</a> <a href=https://github.com/cortexproject/cortex/pull/3625>#3625</a><ul><li><code>cortex_bucket_index_loads_total</code></li><li><code>cortex_bucket_index_load_failures_total</code></li><li><code>cortex_bucket_index_load_duration_seconds</code></li><li><code>cortex_bucket_index_loaded</code></li></ul></li><li>[ENHANCEMENT] Compactor: exported the following metrics. <a href=https://github.com/cortexproject/cortex/pull/3583>#3583</a> <a href=https://github.com/cortexproject/cortex/pull/3625>#3625</a><ul><li><code>cortex_bucket_blocks_count</code>: Total number of blocks per tenant in the bucket. Includes blocks marked for deletion, but not partial blocks.</li><li><code>cortex_bucket_blocks_marked_for_deletion_count</code>: Total number of blocks per tenant marked for deletion in the bucket.</li><li><code>cortex_bucket_blocks_partials_count</code>: Total number of partial blocks.</li><li><code>cortex_bucket_index_last_successful_update_timestamp_seconds</code>: Timestamp of the last successful update of a tenant&rsquo;s bucket index.</li></ul></li><li>[ENHANCEMENT] Ruler: Add <code>cortex_prometheus_last_evaluation_samples</code> to expose the number of samples generated by a rule group per tenant. <a href=https://github.com/cortexproject/cortex/pull/3582>#3582</a></li><li>[ENHANCEMENT] Memberlist: add status page (/memberlist) with available details about memberlist-based KV store and memberlist cluster. It&rsquo;s also possible to view KV values in Go struct or JSON format, or download for inspection. <a href=https://github.com/cortexproject/cortex/pull/3575>#3575</a></li><li>[ENHANCEMENT] Memberlist: client can now keep a size-bounded buffer with sent and received messages and display them in the admin UI (/memberlist) for troubleshooting. <a href=https://github.com/cortexproject/cortex/pull/3581>#3581</a> <a href=https://github.com/cortexproject/cortex/pull/3602>#3602</a></li><li>[ENHANCEMENT] Blocks storage: added block index attributes caching support to metadata cache. The TTL can be configured via <code>-blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl</code>. <a href=https://github.com/cortexproject/cortex/pull/3629>#3629</a></li><li>[ENHANCEMENT] Alertmanager: Add support for Azure blob storage. <a href=https://github.com/cortexproject/cortex/pull/3634>#3634</a></li><li>[ENHANCEMENT] Compactor: tenants marked for deletion will now be fully cleaned up after some delay since deletion of last block. Cleanup includes removal of remaining marker files (including tenant deletion mark file) and files under <code>debug/metas</code>. <a href=https://github.com/cortexproject/cortex/pull/3613>#3613</a></li><li>[ENHANCEMENT] Compactor: retry compaction of a single tenant on failure instead of re-running compaction for all tenants. <a href=https://github.com/cortexproject/cortex/pull/3627>#3627</a></li><li>[ENHANCEMENT] Querier: Implement result caching for tenant query federation. <a href=https://github.com/cortexproject/cortex/pull/3640>#3640</a></li><li>[ENHANCEMENT] API: Add a <code>mode</code> query parameter for the config endpoint: <a href=https://github.com/cortexproject/cortex/pull/3645>#3645</a><ul><li><code>/config?mode=diff</code>: Shows the YAML configuration with all values that differ from the defaults.</li><li><code>/config?mode=defaults</code>: Shows the YAML configuration with all the default values.</li></ul></li><li>[ENHANCEMENT] OpenStack Swift: added the following config options to OpenStack Swift backend client: <a href=https://github.com/cortexproject/cortex/pull/3660>#3660</a><ul><li>Chunks storage: <code>-swift.auth-version</code>, <code>-swift.max-retries</code>, <code>-swift.connect-timeout</code>, <code>-swift.request-timeout</code>.</li><li>Blocks storage: <code>-blocks-storage.swift.auth-version</code>, <code>-blocks-storage.swift.max-retries</code>, <code>-blocks-storage.swift.connect-timeout</code>, <code>-blocks-storage.swift.request-timeout</code>.</li><li>Ruler: <code>-ruler.storage.swift.auth-version</code>, <code>-ruler.storage.swift.max-retries</code>, <code>-ruler.storage.swift.connect-timeout</code>, <code>-ruler.storage.swift.request-timeout</code>.</li></ul></li><li>[ENHANCEMENT] Disabled in-memory shuffle-sharding subring cache in the store-gateway, ruler and compactor. This should reduce the memory utilisation in these services when shuffle-sharding is enabled, without introducing a significantly increase CPU utilisation. <a href=https://github.com/cortexproject/cortex/pull/3601>#3601</a></li><li>[ENHANCEMENT] Shuffle sharding: optimised subring generation used by shuffle sharding. <a href=https://github.com/cortexproject/cortex/pull/3601>#3601</a></li><li>[ENHANCEMENT] New /runtime_config endpoint that returns the defined runtime configuration in YAML format. The returned configuration includes overrides. <a href=https://github.com/cortexproject/cortex/pull/3639>#3639</a></li><li>[ENHANCEMENT] Query-frontend: included the parameter name failed to validate in HTTP 400 message. <a href=https://github.com/cortexproject/cortex/pull/3703>#3703</a></li><li>[ENHANCEMENT] Fail to startup Cortex if provided runtime config is invalid. <a href=https://github.com/cortexproject/cortex/pull/3707>#3707</a></li><li>[ENHANCEMENT] Alertmanager: Add flags to customize the cluster configuration: <a href=https://github.com/cortexproject/cortex/pull/3667>#3667</a><ul><li><code>-alertmanager.cluster.gossip-interval</code>: The interval between sending gossip messages. By lowering this value (more frequent) gossip messages are propagated across cluster more quickly at the expense of increased bandwidth usage.</li><li><code>-alertmanager.cluster.push-pull-interval</code>: The interval between gossip state syncs. Setting this interval lower (more frequent) will increase convergence speeds across larger clusters at the expense of increased bandwidth usage.</li></ul></li><li>[ENHANCEMENT] Distributor: change the error message returned when a received series has too many label values. The new message format has the series at the end and this plays better with Prometheus logs truncation. <a href=https://github.com/cortexproject/cortex/pull/3718>#3718</a><ul><li>From: <code>sample for '&lt;series>' has &lt;value> label names; limit &lt;value></code></li><li>To: <code>series has too many labels (actual: &lt;value>, limit: &lt;value>) series: '&lt;series>'</code></li></ul></li><li>[ENHANCEMENT] Improve bucket index loader to handle edge case where new tenant has not had blocks uploaded to storage yet. <a href=https://github.com/cortexproject/cortex/pull/3717>#3717</a></li><li>[BUGFIX] Allow <code>-querier.max-query-lookback</code> use <code>y|w|d</code> suffix like deprecated <code>-store.max-look-back-period</code>. <a href=https://github.com/cortexproject/cortex/pull/3598>#3598</a></li><li>[BUGFIX] Memberlist: Entry in the ring should now not appear again after using &ldquo;Forget&rdquo; feature (unless it&rsquo;s still heartbeating). <a href=https://github.com/cortexproject/cortex/pull/3603>#3603</a></li><li>[BUGFIX] Ingester: do not close idle TSDBs while blocks shipping is in progress. <a href=https://github.com/cortexproject/cortex/pull/3630>#3630</a> <a href=https://github.com/cortexproject/cortex/pull/3632>#3632</a></li><li>[BUGFIX] Ingester: correctly update <code>cortex_ingester_memory_users</code> and <code>cortex_ingester_active_series</code> when a tenant&rsquo;s idle TSDB is closed, when running Cortex with the blocks storage. <a href=https://github.com/cortexproject/cortex/pull/3646>#3646</a></li><li>[BUGFIX] Querier: fix default value incorrectly overriding <code>-querier.frontend-address</code> in single-binary mode. <a href=https://github.com/cortexproject/cortex/pull/3650>#3650</a></li><li>[BUGFIX] Compactor: delete <code>deletion-mark.json</code> at last when deleting a block in order to not leave partial blocks without deletion mark in the bucket if the compactor is interrupted while deleting a block. <a href=https://github.com/cortexproject/cortex/pull/3660>#3660</a></li><li>[BUGFIX] Blocks storage: do not cleanup a partially uploaded block when <code>meta.json</code> upload fails. Despite failure to upload <code>meta.json</code>, this file may in some cases still appear in the bucket later. By skipping early cleanup, we avoid having corrupted blocks in the storage. <a href=https://github.com/cortexproject/cortex/pull/3660>#3660</a></li><li>[BUGFIX] Alertmanager: disable access to <code>/alertmanager/metrics</code> (which exposes all Cortex metrics), <code>/alertmanager/-/reload</code> and <code>/alertmanager/debug/*</code>, which were available to any authenticated user with enabled AlertManager. <a href=https://github.com/cortexproject/cortex/pull/3678>#3678</a></li><li>[BUGFIX] Query-Frontend: avoid creating many small sub-queries by discarding cache extents under 5 minutes <a href=https://github.com/cortexproject/cortex/pull/3653>#3653</a></li><li>[BUGFIX] Ruler: Ensure the stale markers generated for evaluated rules respect the configured <code>-ruler.evaluation-delay-duration</code>. This will avoid issues with samples with NaN be persisted with timestamps set ahead of the next rule evaluation. <a href=https://github.com/cortexproject/cortex/pull/3687>#3687</a></li><li>[BUGFIX] Alertmanager: don&rsquo;t serve HTTP requests until Alertmanager has fully started. Serving HTTP requests earlier may result in loss of configuration for the user. <a href=https://github.com/cortexproject/cortex/pull/3679>#3679</a></li><li>[BUGFIX] Do not log &ldquo;failed to load config&rdquo; if runtime config file is empty. <a href=https://github.com/cortexproject/cortex/pull/3706>#3706</a></li><li>[BUGFIX] Do not allow to use a runtime config file containing multiple YAML documents. <a href=https://github.com/cortexproject/cortex/pull/3706>#3706</a></li><li>[BUGFIX] HA Tracker: don&rsquo;t track as error in the <code>cortex_kv_request_duration_seconds</code> metric a CAS operation intentionally aborted. <a href=https://github.com/cortexproject/cortex/pull/3745>#3745</a></li></ul><h2 id=160--2020-12-29>1.6.0 / 2020-12-29</h2><ul><li>[CHANGE] Query Frontend: deprecate <code>-querier.compress-http-responses</code> in favour of <code>-api.response-compression-enabled</code>. <a href=https://github.com/cortexproject/cortex/pull/3544>#3544</a></li><li>[CHANGE] Querier: deprecated <code>-store.max-look-back-period</code>. You should use <code>-querier.max-query-lookback</code> instead. <a href=https://github.com/cortexproject/cortex/pull/3452>#3452</a></li><li>[CHANGE] Blocks storage: increased <code>-blocks-storage.bucket-store.chunks-cache.attributes-ttl</code> default from <code>24h</code> to <code>168h</code> (1 week). <a href=https://github.com/cortexproject/cortex/pull/3528>#3528</a></li><li>[CHANGE] Blocks storage: the config option <code>-blocks-storage.bucket-store.index-cache.postings-compression-enabled</code> has been deprecated and postings compression is always enabled. <a href=https://github.com/cortexproject/cortex/pull/3538>#3538</a></li><li>[CHANGE] Ruler: gRPC message size default limits on the Ruler-client side have changed: <a href=https://github.com/cortexproject/cortex/pull/3523>#3523</a><ul><li>limit for outgoing gRPC messages has changed from 2147483647 to 16777216 bytes</li><li>limit for incoming gRPC messages has changed from 4194304 to 104857600 bytes</li></ul></li><li>[FEATURE] Distributor/Ingester: Provide ability to not overflow writes in the presence of a leaving or unhealthy ingester. This allows for more efficient ingester rolling restarts. <a href=https://github.com/cortexproject/cortex/pull/3305>#3305</a></li><li>[FEATURE] Query-frontend: introduced query statistics logged in the query-frontend when enabled via <code>-frontend.query-stats-enabled=true</code>. When enabled, the metric <code>cortex_query_seconds_total</code> is tracked, counting the sum of the wall time spent across all queriers while running queries (on a per-tenant basis). The metrics <code>cortex_request_duration_seconds</code> and <code>cortex_query_seconds_total</code> are different: the first one tracks the request duration (eg. HTTP request from the client), while the latter tracks the sum of the wall time on all queriers involved executing the query. <a href=https://github.com/cortexproject/cortex/pull/3539>#3539</a></li><li>[ENHANCEMENT] API: Add GZIP HTTP compression to the API responses. Compression can be enabled via <code>-api.response-compression-enabled</code>. <a href=https://github.com/cortexproject/cortex/pull/3536>#3536</a></li><li>[ENHANCEMENT] Added zone-awareness support on queries. When zone-awareness is enabled, queries will still succeed if all ingesters in a single zone will fail. <a href=https://github.com/cortexproject/cortex/pull/3414>#3414</a></li><li>[ENHANCEMENT] Blocks storage ingester: exported more TSDB-related metrics. <a href=https://github.com/cortexproject/cortex/pull/3412>#3412</a><ul><li><code>cortex_ingester_tsdb_wal_corruptions_total</code></li><li><code>cortex_ingester_tsdb_head_truncations_failed_total</code></li><li><code>cortex_ingester_tsdb_head_truncations_total</code></li><li><code>cortex_ingester_tsdb_head_gc_duration_seconds</code></li></ul></li><li>[ENHANCEMENT] Enforced keepalive on all gRPC clients used for inter-service communication. <a href=https://github.com/cortexproject/cortex/pull/3431>#3431</a></li><li>[ENHANCEMENT] Added <code>cortex_alertmanager_config_hash</code> metric to expose hash of Alertmanager Config loaded per user. <a href=https://github.com/cortexproject/cortex/pull/3388>#3388</a></li><li>[ENHANCEMENT] Query-Frontend / Query-Scheduler: New component called &ldquo;Query-Scheduler&rdquo; has been introduced. Query-Scheduler is simply a queue of requests, moved outside of Query-Frontend. This allows Query-Frontend to be scaled separately from number of queues. To make Query-Frontend and Querier use Query-Scheduler, they need to be started with <code>-frontend.scheduler-address</code> and <code>-querier.scheduler-address</code> options respectively. <a href=https://github.com/cortexproject/cortex/pull/3374>#3374</a> <a href=https://github.com/cortexproject/cortex/pull/3471>#3471</a></li><li>[ENHANCEMENT] Query-frontend / Querier / Ruler: added <code>-querier.max-query-lookback</code> to limit how long back data (series and metadata) can be queried. This setting can be overridden on a per-tenant basis and is enforced in the query-frontend, querier and ruler. <a href=https://github.com/cortexproject/cortex/pull/3452>#3452</a> <a href=https://github.com/cortexproject/cortex/pull/3458>#3458</a></li><li>[ENHANCEMENT] Querier: added <code>-querier.query-store-for-labels-enabled</code> to query store for label names, label values and series APIs. Only works with blocks storage engine. <a href=https://github.com/cortexproject/cortex/pull/3461>#3461</a> <a href=https://github.com/cortexproject/cortex/pull/3520>#3520</a></li><li>[ENHANCEMENT] Ingester: exposed <code>-blocks-storage.tsdb.wal-segment-size-bytes</code> config option to customise the TSDB WAL segment max size. <a href=https://github.com/cortexproject/cortex/pull/3476>#3476</a></li><li>[ENHANCEMENT] Compactor: concurrently run blocks cleaner for multiple tenants. Concurrency can be configured via <code>-compactor.cleanup-concurrency</code>. <a href=https://github.com/cortexproject/cortex/pull/3483>#3483</a></li><li>[ENHANCEMENT] Compactor: shuffle tenants before running compaction. <a href=https://github.com/cortexproject/cortex/pull/3483>#3483</a></li><li>[ENHANCEMENT] Compactor: wait for a stable ring at startup, when sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/3484>#3484</a></li><li>[ENHANCEMENT] Store-gateway: added <code>-blocks-storage.bucket-store.index-header-lazy-loading-enabled</code> to enable index-header lazy loading (experimental). When enabled, index-headers will be mmap-ed only once required by a query and will be automatically released after <code>-blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout</code> time of inactivity. <a href=https://github.com/cortexproject/cortex/pull/3498>#3498</a></li><li>[ENHANCEMENT] Alertmanager: added metrics <code>cortex_alertmanager_notification_requests_total</code> and <code>cortex_alertmanager_notification_requests_failed_total</code>. <a href=https://github.com/cortexproject/cortex/pull/3518>#3518</a></li><li>[ENHANCEMENT] Ingester: added <code>-blocks-storage.tsdb.head-chunks-write-buffer-size-bytes</code> to fine-tune the TSDB head chunks write buffer size when running Cortex blocks storage. <a href=https://github.com/cortexproject/cortex/pull/3518>#3518</a></li><li>[ENHANCEMENT] /metrics now supports OpenMetrics output. HTTP and gRPC servers metrics can now include exemplars. <a href=https://github.com/cortexproject/cortex/pull/3524>#3524</a></li><li>[ENHANCEMENT] Expose gRPC keepalive policy options by gRPC server. <a href=https://github.com/cortexproject/cortex/pull/3524>#3524</a></li><li>[ENHANCEMENT] Blocks storage: enabled caching of <code>meta.json</code> attributes, configurable via <code>-blocks-storage.bucket-store.metadata-cache.metafile-attributes-ttl</code>. <a href=https://github.com/cortexproject/cortex/pull/3528>#3528</a></li><li>[ENHANCEMENT] Compactor: added a config validation check to fail fast if the compactor has been configured invalid block range periods (each period is expected to be a multiple of the previous one). <a href=https://github.com/cortexproject/cortex/pull/3534>#3534</a></li><li>[ENHANCEMENT] Blocks storage: concurrently fetch deletion marks from object storage. <a href=https://github.com/cortexproject/cortex/pull/3538>#3538</a></li><li>[ENHANCEMENT] Blocks storage ingester: ingester can now close idle TSDB and delete local data. <a href=https://github.com/cortexproject/cortex/pull/3491>#3491</a> <a href=https://github.com/cortexproject/cortex/pull/3552>#3552</a></li><li>[ENHANCEMENT] Blocks storage: add option to use V2 signatures for S3 authentication. <a href=https://github.com/cortexproject/cortex/pull/3540>#3540</a></li><li>[ENHANCEMENT] Exported process metrics to monitor the number of memory map areas allocated. <a href=https://github.com/cortexproject/cortex/pull/3537>#3537</a><ul><li><ul><li><code>process_memory_map_areas</code></li></ul></li><li><ul><li><code>process_memory_map_areas_limit</code></li></ul></li></ul></li><li>[ENHANCEMENT] Ruler: Expose gRPC client options. <a href=https://github.com/cortexproject/cortex/pull/3523>#3523</a></li><li>[ENHANCEMENT] Compactor: added metrics to track on-going compaction. <a href=https://github.com/cortexproject/cortex/pull/3535>#3535</a><ul><li><code>cortex_compactor_tenants_discovered</code></li><li><code>cortex_compactor_tenants_skipped</code></li><li><code>cortex_compactor_tenants_processing_succeeded</code></li><li><code>cortex_compactor_tenants_processing_failed</code></li></ul></li><li>[ENHANCEMENT] Added new experimental API endpoints: <code>POST /purger/delete_tenant</code> and <code>GET /purger/delete_tenant_status</code> for deleting all tenant data. Only works with blocks storage. Compactor removes blocks that belong to user marked for deletion. <a href=https://github.com/cortexproject/cortex/pull/3549>#3549</a> <a href=https://github.com/cortexproject/cortex/pull/3558>#3558</a></li><li>[ENHANCEMENT] Chunks storage: add option to use V2 signatures for S3 authentication. <a href=https://github.com/cortexproject/cortex/pull/3560>#3560</a></li><li>[ENHANCEMENT] HA Tracker: Added new limit <code>ha_max_clusters</code> to set the max number of clusters tracked for single user. This limit is disabled by default. <a href=https://github.com/cortexproject/cortex/pull/3668>#3668</a></li><li>[BUGFIX] Query-Frontend: <code>cortex_query_seconds_total</code> now return seconds not nanoseconds. <a href=https://github.com/cortexproject/cortex/pull/3589>#3589</a></li><li>[BUGFIX] Blocks storage ingester: fixed some cases leading to a TSDB WAL corruption after a partial write to disk. <a href=https://github.com/cortexproject/cortex/pull/3423>#3423</a></li><li>[BUGFIX] Blocks storage: Fix the race between ingestion and <code>/flush</code> call resulting in overlapping blocks. <a href=https://github.com/cortexproject/cortex/pull/3422>#3422</a></li><li>[BUGFIX] Querier: fixed <code>-querier.max-query-into-future</code> which wasn&rsquo;t correctly enforced on range queries. <a href=https://github.com/cortexproject/cortex/pull/3452>#3452</a></li><li>[BUGFIX] Fixed float64 precision stability when aggregating metrics before exposing them. This could have lead to false counters resets when querying some metrics exposed by Cortex. <a href=https://github.com/cortexproject/cortex/pull/3506>#3506</a></li><li>[BUGFIX] Querier: the meta.json sync concurrency done when running Cortex with the blocks storage is now controlled by <code>-blocks-storage.bucket-store.meta-sync-concurrency</code> instead of the incorrect <code>-blocks-storage.bucket-store.block-sync-concurrency</code> (default values are the same). <a href=https://github.com/cortexproject/cortex/pull/3531>#3531</a></li><li>[BUGFIX] Querier: fixed initialization order of querier module when using blocks storage. It now (again) waits until blocks have been synchronized. <a href=https://github.com/cortexproject/cortex/pull/3551>#3551</a></li></ul><h2 id=blocksconvert-2>Blocksconvert</h2><ul><li>[ENHANCEMENT] Scheduler: ability to ignore users based on regexp, using <code>-scheduler.ignore-users-regex</code> flag. <a href=https://github.com/cortexproject/cortex/pull/3477>#3477</a></li><li>[ENHANCEMENT] Builder: Parallelize reading chunks in the final stage of building block. <a href=https://github.com/cortexproject/cortex/pull/3470>#3470</a></li><li>[ENHANCEMENT] Builder: remove duplicate label names from chunk. <a href=https://github.com/cortexproject/cortex/pull/3547>#3547</a></li></ul><h2 id=150--2020-11-09>1.5.0 / 2020-11-09</h2><h3 id=cortex>Cortex</h3><ul><li>[CHANGE] Blocks storage: update the default HTTP configuration values for the S3 client to the upstream Thanos default values. <a href=https://github.com/cortexproject/cortex/pull/3244>#3244</a><ul><li><code>-blocks-storage.s3.http.idle-conn-timeout</code> is set 90 seconds.</li><li><code>-blocks-storage.s3.http.response-header-timeout</code> is set to 2 minutes.</li></ul></li><li>[CHANGE] Improved shuffle sharding support in the write path. This work introduced some config changes: <a href=https://github.com/cortexproject/cortex/pull/3090>#3090</a><ul><li>Introduced <code>-distributor.sharding-strategy</code> CLI flag (and its respective <code>sharding_strategy</code> YAML config option) to explicitly specify which sharding strategy should be used in the write path</li><li><code>-experimental.distributor.user-subring-size</code> flag renamed to <code>-distributor.ingestion-tenant-shard-size</code></li><li><code>user_subring_size</code> limit YAML config option renamed to <code>ingestion_tenant_shard_size</code></li></ul></li><li>[CHANGE] Dropped &ldquo;blank Alertmanager configuration; using fallback&rdquo; message from Info to Debug level. <a href=https://github.com/cortexproject/cortex/pull/3205>#3205</a></li><li>[CHANGE] Zone-awareness replication for time-series now should be explicitly enabled in the distributor via the <code>-distributor.zone-awareness-enabled</code> CLI flag (or its respective YAML config option). Before, zone-aware replication was implicitly enabled if a zone was set on ingesters. <a href=https://github.com/cortexproject/cortex/pull/3200>#3200</a></li><li>[CHANGE] Removed the deprecated CLI flag <code>-config-yaml</code>. You should use <code>-schema-config-file</code> instead. <a href=https://github.com/cortexproject/cortex/pull/3225>#3225</a></li><li>[CHANGE] Enforced the HTTP method required by some API endpoints which did (incorrectly) allow any method before that. <a href=https://github.com/cortexproject/cortex/pull/3228>#3228</a><ul><li><code>GET /</code></li><li><code>GET /config</code></li><li><code>GET /debug/fgprof</code></li><li><code>GET /distributor/all_user_stats</code></li><li><code>GET /distributor/ha_tracker</code></li><li><code>GET /all_user_stats</code></li><li><code>GET /ha-tracker</code></li><li><code>GET /api/v1/user_stats</code></li><li><code>GET /api/v1/chunks</code></li><li><code>GET &lt;legacy-http-prefix>/user_stats</code></li><li><code>GET &lt;legacy-http-prefix>/chunks</code></li><li><code>GET /services</code></li><li><code>GET /multitenant_alertmanager/status</code></li><li><code>GET /status</code> (alertmanager microservice)</li><li><code>GET|POST /ingester/ring</code></li><li><code>GET|POST /ring</code></li><li><code>GET|POST /store-gateway/ring</code></li><li><code>GET|POST /compactor/ring</code></li><li><code>GET|POST /ingester/flush</code></li><li><code>GET|POST /ingester/shutdown</code></li><li><code>GET|POST /flush</code></li><li><code>GET|POST /shutdown</code></li><li><code>GET|POST /ruler/ring</code></li><li><code>POST /api/v1/push</code></li><li><code>POST &lt;legacy-http-prefix>/push</code></li><li><code>POST /push</code></li><li><code>POST /ingester/push</code></li></ul></li><li>[CHANGE] Renamed CLI flags to configure the network interface names from which automatically detect the instance IP. <a href=https://github.com/cortexproject/cortex/pull/3295>#3295</a><ul><li><code>-compactor.ring.instance-interface</code> renamed to <code>-compactor.ring.instance-interface-names</code></li><li><code>-store-gateway.sharding-ring.instance-interface</code> renamed to <code>-store-gateway.sharding-ring.instance-interface-names</code></li><li><code>-distributor.ring.instance-interface</code> renamed to <code>-distributor.ring.instance-interface-names</code></li><li><code>-ruler.ring.instance-interface</code> renamed to <code>-ruler.ring.instance-interface-names</code></li></ul></li><li>[CHANGE] Renamed <code>-&lt;prefix>.redis.enable-tls</code> CLI flag to <code>-&lt;prefix>.redis.tls-enabled</code>, and its respective YAML config option from <code>enable_tls</code> to <code>tls_enabled</code>. <a href=https://github.com/cortexproject/cortex/pull/3298>#3298</a></li><li>[CHANGE] Increased default <code>-&lt;prefix>.redis.timeout</code> from <code>100ms</code> to <code>500ms</code>. <a href=https://github.com/cortexproject/cortex/pull/3301>#3301</a></li><li>[CHANGE] <code>cortex_alertmanager_config_invalid</code> has been removed in favor of <code>cortex_alertmanager_config_last_reload_successful</code>. <a href=https://github.com/cortexproject/cortex/pull/3289>#3289</a></li><li>[CHANGE] Query-frontend: POST requests whose body size exceeds 10MiB will be rejected. The max body size can be customised via <code>-frontend.max-body-size</code>. <a href=https://github.com/cortexproject/cortex/pull/3276>#3276</a></li><li>[FEATURE] Shuffle sharding: added support for shuffle-sharding queriers in the query-frontend. When configured (<code>-frontend.max-queriers-per-tenant</code> globally, or using per-tenant limit <code>max_queriers_per_tenant</code>), each tenants&rsquo;s requests will be handled by different set of queriers. <a href=https://github.com/cortexproject/cortex/pull/3113>#3113</a> <a href=https://github.com/cortexproject/cortex/pull/3257>#3257</a></li><li>[FEATURE] Shuffle sharding: added support for shuffle-sharding ingesters on the read path. When ingesters shuffle-sharding is enabled and <code>-querier.shuffle-sharding-ingesters-lookback-period</code> is set, queriers will fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since &rsquo;now - lookback period&rsquo;. <a href=https://github.com/cortexproject/cortex/pull/3252>#3252</a></li><li>[FEATURE] Query-frontend: added <code>compression</code> config to support results cache with compression. <a href=https://github.com/cortexproject/cortex/pull/3217>#3217</a></li><li>[FEATURE] Add OpenStack Swift support to blocks storage. <a href=https://github.com/cortexproject/cortex/pull/3303>#3303</a></li><li>[FEATURE] Added support for applying Prometheus relabel configs on series received by the distributor. A <code>metric_relabel_configs</code> field has been added to the per-tenant limits configuration. <a href=https://github.com/cortexproject/cortex/pull/3329>#3329</a></li><li>[FEATURE] Support for Cassandra client SSL certificates. <a href=https://github.com/cortexproject/cortex/pull/3384>#3384</a></li><li>[ENHANCEMENT] Ruler: Introduces two new limits <code>-ruler.max-rules-per-rule-group</code> and <code>-ruler.max-rule-groups-per-tenant</code> to control the number of rules per rule group and the total number of rule groups for a given user. They are disabled by default. <a href=https://github.com/cortexproject/cortex/pull/3366>#3366</a></li><li>[ENHANCEMENT] Allow to specify multiple comma-separated Cortex services to <code>-target</code> CLI option (or its respective YAML config option). For example, <code>-target=all,compactor</code> can be used to start Cortex single-binary with compactor as well. <a href=https://github.com/cortexproject/cortex/pull/3275>#3275</a></li><li>[ENHANCEMENT] Expose additional HTTP configs for the S3 backend client. New flag are listed below: <a href=https://github.com/cortexproject/cortex/pull/3244>#3244</a><ul><li><code>-blocks-storage.s3.http.idle-conn-timeout</code></li><li><code>-blocks-storage.s3.http.response-header-timeout</code></li><li><code>-blocks-storage.s3.http.insecure-skip-verify</code></li></ul></li><li>[ENHANCEMENT] Added <code>cortex_query_frontend_connected_clients</code> metric to show the number of workers currently connected to the frontend. <a href=https://github.com/cortexproject/cortex/pull/3207>#3207</a></li><li>[ENHANCEMENT] Shuffle sharding: improved shuffle sharding in the write path. Shuffle sharding now should be explicitly enabled via <code>-distributor.sharding-strategy</code> CLI flag (or its respective YAML config option) and guarantees stability, consistency, shuffling and balanced zone-awareness properties. <a href=https://github.com/cortexproject/cortex/pull/3090>#3090</a> <a href=https://github.com/cortexproject/cortex/pull/3214>#3214</a></li><li>[ENHANCEMENT] Ingester: added new metric <code>cortex_ingester_active_series</code> to track active series more accurately. Also added options to control whether active series tracking is enabled (<code>-ingester.active-series-metrics-enabled</code>, defaults to false), and how often this metric is updated (<code>-ingester.active-series-metrics-update-period</code>) and max idle time for series to be considered inactive (<code>-ingester.active-series-metrics-idle-timeout</code>). <a href=https://github.com/cortexproject/cortex/pull/3153>#3153</a></li><li>[ENHANCEMENT] Store-gateway: added zone-aware replication support to blocks replication in the store-gateway. <a href=https://github.com/cortexproject/cortex/pull/3200>#3200</a></li><li>[ENHANCEMENT] Store-gateway: exported new metrics. <a href=https://github.com/cortexproject/cortex/pull/3231>#3231</a><ul><li><code>cortex_bucket_store_cached_series_fetch_duration_seconds</code></li><li><code>cortex_bucket_store_cached_postings_fetch_duration_seconds</code></li><li><code>cortex_bucket_stores_gate_queries_max</code></li></ul></li><li>[ENHANCEMENT] Added <code>-version</code> flag to Cortex. <a href=https://github.com/cortexproject/cortex/pull/3233>#3233</a></li><li>[ENHANCEMENT] Hash ring: added instance registered timestamp to the ring. <a href=https://github.com/cortexproject/cortex/pull/3248>#3248</a></li><li>[ENHANCEMENT] Reduce tail latency by smoothing out spikes in rate of chunk flush operations. <a href=https://github.com/cortexproject/cortex/pull/3191>#3191</a></li><li>[ENHANCEMENT] User Cortex as User Agent in http requests issued by Configs DB client. <a href=https://github.com/cortexproject/cortex/pull/3264>#3264</a></li><li>[ENHANCEMENT] Experimental Ruler API: Fetch rule groups from object storage in parallel. <a href=https://github.com/cortexproject/cortex/pull/3218>#3218</a></li><li>[ENHANCEMENT] Chunks GCS object storage client uses the <code>fields</code> selector to limit the payload size when listing objects in the bucket. <a href=https://github.com/cortexproject/cortex/pull/3218>#3218</a> <a href=https://github.com/cortexproject/cortex/pull/3292>#3292</a></li><li>[ENHANCEMENT] Added shuffle sharding support to ruler. Added new metric <code>cortex_ruler_sync_rules_total</code>. <a href=https://github.com/cortexproject/cortex/pull/3235>#3235</a></li><li>[ENHANCEMENT] Return an explicit error when the store-gateway is explicitly requested without a blocks storage engine. <a href=https://github.com/cortexproject/cortex/pull/3287>#3287</a></li><li>[ENHANCEMENT] Ruler: only load rules that belong to the ruler. Improves rules syncing performances when ruler sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/3269>#3269</a></li><li>[ENHANCEMENT] Added <code>-&lt;prefix>.redis.tls-insecure-skip-verify</code> flag. <a href=https://github.com/cortexproject/cortex/pull/3298>#3298</a></li><li>[ENHANCEMENT] Added <code>cortex_alertmanager_config_last_reload_successful_seconds</code> metric to show timestamp of last successful AM config reload. <a href=https://github.com/cortexproject/cortex/pull/3289>#3289</a></li><li>[ENHANCEMENT] Blocks storage: reduced number of bucket listing operations to list block content (applies to newly created blocks only). <a href=https://github.com/cortexproject/cortex/pull/3363>#3363</a></li><li>[ENHANCEMENT] Ruler: Include the tenant ID on the notifier logs. <a href=https://github.com/cortexproject/cortex/pull/3372>#3372</a></li><li>[ENHANCEMENT] Blocks storage Compactor: Added <code>-compactor.enabled-tenants</code> and <code>-compactor.disabled-tenants</code> to explicitly enable or disable compaction of specific tenants. <a href=https://github.com/cortexproject/cortex/pull/3385>#3385</a></li><li>[ENHANCEMENT] Blocks storage ingester: Creating checkpoint only once even when there are multiple Head compactions in a single <code>Compact()</code> call. <a href=https://github.com/cortexproject/cortex/pull/3373>#3373</a></li><li>[BUGFIX] Blocks storage ingester: Read repair memory-mapped chunks file which can end up being empty on abrupt shutdowns combined with faulty disks. <a href=https://github.com/cortexproject/cortex/pull/3373>#3373</a></li><li>[BUGFIX] Blocks storage ingester: Close TSDB resources on failed startup preventing ingester OOMing. <a href=https://github.com/cortexproject/cortex/pull/3373>#3373</a></li><li>[BUGFIX] No-longer-needed ingester operations for queries triggered by queriers and rulers are now canceled. <a href=https://github.com/cortexproject/cortex/pull/3178>#3178</a></li><li>[BUGFIX] Ruler: directories in the configured <code>rules-path</code> will be removed on startup and shutdown in order to ensure they don&rsquo;t persist between runs. <a href=https://github.com/cortexproject/cortex/pull/3195>#3195</a></li><li>[BUGFIX] Handle hash-collisions in the query path. <a href=https://github.com/cortexproject/cortex/pull/3192>#3192</a></li><li>[BUGFIX] Check for postgres rows errors. <a href=https://github.com/cortexproject/cortex/pull/3197>#3197</a></li><li>[BUGFIX] Ruler Experimental API: Don&rsquo;t allow rule groups without names or empty rule groups. <a href=https://github.com/cortexproject/cortex/pull/3210>#3210</a></li><li>[BUGFIX] Experimental Alertmanager API: Do not allow empty Alertmanager configurations or bad template filenames to be submitted through the configuration API. <a href=https://github.com/cortexproject/cortex/pull/3185>#3185</a></li><li>[BUGFIX] Reduce failures to update heartbeat when using Consul. <a href=https://github.com/cortexproject/cortex/pull/3259>#3259</a></li><li>[BUGFIX] When using ruler sharding, moving all user rule groups from ruler to a different one and then back could end up with some user groups not being evaluated at all. <a href=https://github.com/cortexproject/cortex/pull/3235>#3235</a></li><li>[BUGFIX] Fixed shuffle sharding consistency when zone-awareness is enabled and the shard size is increased or instances in a new zone are added. <a href=https://github.com/cortexproject/cortex/pull/3299>#3299</a></li><li>[BUGFIX] Use a valid grpc header when logging IP addresses. <a href=https://github.com/cortexproject/cortex/pull/3307>#3307</a></li><li>[BUGFIX] Fixed the metric <code>cortex_prometheus_rule_group_duration_seconds</code> in the Ruler, it wouldn&rsquo;t report any values. <a href=https://github.com/cortexproject/cortex/pull/3310>#3310</a></li><li>[BUGFIX] Fixed gRPC connections leaking in rulers when rulers sharding is enabled and APIs called. <a href=https://github.com/cortexproject/cortex/pull/3314>#3314</a></li><li>[BUGFIX] Fixed shuffle sharding consistency when zone-awareness is enabled and the shard size is increased or instances in a new zone are added. <a href=https://github.com/cortexproject/cortex/pull/3299>#3299</a></li><li>[BUGFIX] Fixed Gossip memberlist members joining when addresses are configured using DNS-based service discovery. <a href=https://github.com/cortexproject/cortex/pull/3360>#3360</a></li><li>[BUGFIX] Ingester: fail to start an ingester running the blocks storage, if unable to load any existing TSDB at startup. <a href=https://github.com/cortexproject/cortex/pull/3354>#3354</a></li><li>[BUGFIX] Blocks storage: Avoid deletion of blocks in the ingester which are not shipped to the storage yet. <a href=https://github.com/cortexproject/cortex/pull/3346>#3346</a></li><li>[BUGFIX] Fix common prefixes returned by List method of S3 client. <a href=https://github.com/cortexproject/cortex/pull/3358>#3358</a></li><li>[BUGFIX] Honor configured timeout in Azure and GCS object clients. <a href=https://github.com/cortexproject/cortex/pull/3285>#3285</a></li><li>[BUGFIX] Blocks storage: Avoid creating blocks larger than configured block range period on forced compaction and when TSDB is idle. <a href=https://github.com/cortexproject/cortex/pull/3344>#3344</a></li><li>[BUGFIX] Shuffle sharding: fixed max global series per user/metric limit when shuffle sharding and <code>-distributor.shard-by-all-labels=true</code> are both enabled in distributor. When using these global limits you should now set <code>-distributor.sharding-strategy</code> and <code>-distributor.zone-awareness-enabled</code> to ingesters too. <a href=https://github.com/cortexproject/cortex/pull/3369>#3369</a></li><li>[BUGFIX] Slow query logging: when using downstream server request parameters were not logged. <a href=https://github.com/cortexproject/cortex/pull/3276>#3276</a></li><li>[BUGFIX] Fixed tenant detection in the ruler and alertmanager API when running without auth. <a href=https://github.com/cortexproject/cortex/pull/3343>#3343</a></li></ul><h3 id=blocksconvert-3>Blocksconvert</h3><ul><li>[ENHANCEMENT] Blocksconvert – Builder: download plan file locally before processing it. <a href=https://github.com/cortexproject/cortex/pull/3209>#3209</a></li><li>[ENHANCEMENT] Blocksconvert – Cleaner: added new tool for deleting chunks data. <a href=https://github.com/cortexproject/cortex/pull/3283>#3283</a></li><li>[ENHANCEMENT] Blocksconvert – Scanner: support for scanning specific date-range only. <a href=https://github.com/cortexproject/cortex/pull/3222>#3222</a></li><li>[ENHANCEMENT] Blocksconvert – Scanner: metrics for tracking progress. <a href=https://github.com/cortexproject/cortex/pull/3222>#3222</a></li><li>[ENHANCEMENT] Blocksconvert – Builder: retry block upload before giving up. <a href=https://github.com/cortexproject/cortex/pull/3245>#3245</a></li><li>[ENHANCEMENT] Blocksconvert – Scanner: upload plans concurrently. <a href=https://github.com/cortexproject/cortex/pull/3340>#3340</a></li><li>[BUGFIX] Blocksconvert: fix chunks ordering in the block. Chunks in different order than series work just fine in TSDB blocks at the moment, but it&rsquo;s not consistent with what Prometheus does and future Prometheus and Cortex optimizations may rely on this ordering. <a href=https://github.com/cortexproject/cortex/pull/3371>#3371</a></li></ul><h2 id=140--2020-10-02>1.4.0 / 2020-10-02</h2><ul><li>[CHANGE] TLS configuration for gRPC, HTTP and etcd clients is now marked as experimental. These features are not yet fully baked, and we expect possible small breaking changes in Cortex 1.5. <a href=https://github.com/cortexproject/cortex/pull/3198>#3198</a></li><li>[CHANGE] Cassandra backend support is now GA (stable). <a href=https://github.com/cortexproject/cortex/pull/3180>#3180</a></li><li>[CHANGE] Blocks storage is now GA (stable). The <code>-experimental</code> prefix has been removed from all CLI flags related to the blocks storage (no YAML config changes). <a href=https://github.com/cortexproject/cortex/pull/3180>#3180</a> <a href=https://github.com/cortexproject/cortex/pull/3201>#3201</a><ul><li><code>-experimental.blocks-storage.*</code> flags renamed to <code>-blocks-storage.*</code></li><li><code>-experimental.store-gateway.*</code> flags renamed to <code>-store-gateway.*</code></li><li><code>-experimental.querier.store-gateway-client.*</code> flags renamed to <code>-querier.store-gateway-client.*</code></li><li><code>-experimental.querier.store-gateway-addresses</code> flag renamed to <code>-querier.store-gateway-addresses</code></li><li><code>-store-gateway.replication-factor</code> flag renamed to <code>-store-gateway.sharding-ring.replication-factor</code></li><li><code>-store-gateway.tokens-file-path</code> flag renamed to <code>store-gateway.sharding-ring.tokens-file-path</code></li></ul></li><li>[CHANGE] Ingester: Removed deprecated untyped record from chunks WAL. Only if you are running <code>v1.0</code> or below, it is recommended to first upgrade to <code>v1.1</code>/<code>v1.2</code>/<code>v1.3</code> and run it for a day before upgrading to <code>v1.4</code> to avoid data loss. <a href=https://github.com/cortexproject/cortex/pull/3115>#3115</a></li><li>[CHANGE] Distributor API endpoints are no longer served unless target is set to <code>distributor</code> or <code>all</code>. <a href=https://github.com/cortexproject/cortex/pull/3112>#3112</a></li><li>[CHANGE] Increase the default Cassandra client replication factor to 3. <a href=https://github.com/cortexproject/cortex/pull/3007>#3007</a></li><li>[CHANGE] Blocks storage: removed the support to transfer blocks between ingesters on shutdown. When running the Cortex blocks storage, ingesters are expected to run with a persistent disk. The following metrics have been removed: <a href=https://github.com/cortexproject/cortex/pull/2996>#2996</a><ul><li><code>cortex_ingester_sent_files</code></li><li><code>cortex_ingester_received_files</code></li><li><code>cortex_ingester_received_bytes_total</code></li><li><code>cortex_ingester_sent_bytes_total</code></li></ul></li><li>[CHANGE] The buckets for the <code>cortex_chunk_store_index_lookups_per_query</code> metric have been changed to 1, 2, 4, 8, 16. <a href=https://github.com/cortexproject/cortex/pull/3021>#3021</a></li><li>[CHANGE] Blocks storage: the <code>operation</code> label value <code>getrange</code> has changed into <code>get_range</code> for the metrics <code>thanos_store_bucket_cache_operation_requests_total</code> and <code>thanos_store_bucket_cache_operation_hits_total</code>. <a href=https://github.com/cortexproject/cortex/pull/3000>#3000</a></li><li>[CHANGE] Experimental Delete Series: <code>/api/v1/admin/tsdb/delete_series</code> and <code>/api/v1/admin/tsdb/cancel_delete_request</code> purger APIs to return status code <code>204</code> instead of <code>200</code> for success. <a href=https://github.com/cortexproject/cortex/pull/2946>#2946</a></li><li>[CHANGE] Histogram <code>cortex_memcache_request_duration_seconds</code> <code>method</code> label value changes from <code>Memcached.Get</code> to <code>Memcached.GetBatched</code> for batched lookups, and is not reported for non-batched lookups (label value <code>Memcached.GetMulti</code> remains, and had exactly the same value as <code>Get</code> in nonbatched lookups). The same change applies to tracing spans. <a href=https://github.com/cortexproject/cortex/pull/3046>#3046</a></li><li>[CHANGE] TLS server validation is now enabled by default, a new parameter <code>tls_insecure_skip_verify</code> can be set to true to skip validation optionally. <a href=https://github.com/cortexproject/cortex/pull/3030>#3030</a></li><li>[CHANGE] <code>cortex_ruler_config_update_failures_total</code> has been removed in favor of <code>cortex_ruler_config_last_reload_successful</code>. <a href=https://github.com/cortexproject/cortex/pull/3056>#3056</a></li><li>[CHANGE] <code>ruler.evaluation_delay_duration</code> field in YAML config has been moved and renamed to <code>limits.ruler_evaluation_delay_duration</code>. <a href=https://github.com/cortexproject/cortex/pull/3098>#3098</a></li><li>[CHANGE] Removed obsolete <code>results_cache.max_freshness</code> from YAML config (deprecated since Cortex 1.2). <a href=https://github.com/cortexproject/cortex/pull/3145>#3145</a></li><li>[CHANGE] Removed obsolete <code>-promql.lookback-delta</code> option (deprecated since Cortex 1.2, replaced with <code>-querier.lookback-delta</code>). <a href=https://github.com/cortexproject/cortex/pull/3144>#3144</a></li><li>[CHANGE] Cache: added support for Redis Cluster and Redis Sentinel. <a href=https://github.com/cortexproject/cortex/pull/2961>#2961</a><ul><li>The following changes have been made in Redis configuration:</li><li><code>-redis.master_name</code> added</li><li><code>-redis.db</code> added</li><li><code>-redis.max-active-conns</code> changed to <code>-redis.pool-size</code></li><li><code>-redis.max-conn-lifetime</code> changed to <code>-redis.max-connection-age</code></li><li><code>-redis.max-idle-conns</code> removed</li><li><code>-redis.wait-on-pool-exhaustion</code> removed</li></ul></li><li>[CHANGE] TLS configuration for gRPC, HTTP and etcd clients is now marked as experimental. These features are not yet fully baked, and we expect possible small breaking changes in Cortex 1.5. <a href=https://github.com/cortexproject/cortex/pull/3198>#3198</a></li><li>[CHANGE] Fixed store-gateway CLI flags inconsistencies. <a href=https://github.com/cortexproject/cortex/pull/3201>#3201</a><ul><li><code>-store-gateway.replication-factor</code> flag renamed to <code>-store-gateway.sharding-ring.replication-factor</code></li><li><code>-store-gateway.tokens-file-path</code> flag renamed to <code>store-gateway.sharding-ring.tokens-file-path</code></li></ul></li><li>[FEATURE] Logging of the source IP passed along by a reverse proxy is now supported by setting the <code>-server.log-source-ips-enabled</code>. For non standard headers the settings <code>-server.log-source-ips-header</code> and <code>-server.log-source-ips-regex</code> can be used. <a href=https://github.com/cortexproject/cortex/pull/2985>#2985</a></li><li>[FEATURE] Blocks storage: added shuffle sharding support to store-gateway blocks sharding. Added the following additional metrics to store-gateway: <a href=https://github.com/cortexproject/cortex/pull/3069>#3069</a><ul><li><code>cortex_bucket_stores_tenants_discovered</code></li><li><code>cortex_bucket_stores_tenants_synced</code></li></ul></li><li>[FEATURE] Experimental blocksconvert: introduce an experimental tool <code>blocksconvert</code> to migrate long-term storage chunks to blocks. <a href=https://github.com/cortexproject/cortex/pull/3092>#3092</a> <a href=https://github.com/cortexproject/cortex/pull/3122>#3122</a> <a href=https://github.com/cortexproject/cortex/pull/3127>#3127</a> <a href=https://github.com/cortexproject/cortex/pull/3162>#3162</a></li><li>[ENHANCEMENT] Improve the Alertmanager logging when serving requests from its API / UI. <a href=https://github.com/cortexproject/cortex/pull/3397>#3397</a></li><li>[ENHANCEMENT] Add support for azure storage in China, German and US Government environments. <a href=https://github.com/cortexproject/cortex/pull/2988>#2988</a></li><li>[ENHANCEMENT] Query-tee: added a small tolerance to floating point sample values comparison. <a href=https://github.com/cortexproject/cortex/pull/2994>#2994</a></li><li>[ENHANCEMENT] Query-tee: add support for doing a passthrough of requests to preferred backend for unregistered routes <a href=https://github.com/cortexproject/cortex/pull/3018>#3018</a></li><li>[ENHANCEMENT] Expose <code>storage.aws.dynamodb.backoff_config</code> configuration file field. <a href=https://github.com/cortexproject/cortex/pull/3026>#3026</a></li><li>[ENHANCEMENT] Added <code>cortex_request_message_bytes</code> and <code>cortex_response_message_bytes</code> histograms to track received and sent gRPC message and HTTP request/response sizes. Added <code>cortex_inflight_requests</code> gauge to track number of inflight gRPC and HTTP requests. <a href=https://github.com/cortexproject/cortex/pull/3064>#3064</a></li><li>[ENHANCEMENT] Publish ruler&rsquo;s ring metrics. <a href=https://github.com/cortexproject/cortex/pull/3074>#3074</a></li><li>[ENHANCEMENT] Add config validation to the experimental Alertmanager API. Invalid configs are no longer accepted. <a href=https://github.com/cortexproject/cortex/pull/3053>#3053</a></li><li>[ENHANCEMENT] Add &ldquo;integration&rdquo; as a label for <code>cortex_alertmanager_notifications_total</code> and <code>cortex_alertmanager_notifications_failed_total</code> metrics. <a href=https://github.com/cortexproject/cortex/pull/3056>#3056</a></li><li>[ENHANCEMENT] Add <code>cortex_ruler_config_last_reload_successful</code> and <code>cortex_ruler_config_last_reload_successful_seconds</code> to check status of users rule manager. <a href=https://github.com/cortexproject/cortex/pull/3056>#3056</a></li><li>[ENHANCEMENT] The configuration validation now fails if an empty YAML node has been set for a root YAML config property. <a href=https://github.com/cortexproject/cortex/pull/3080>#3080</a></li><li>[ENHANCEMENT] Memcached dial() calls now have a circuit-breaker to avoid hammering a broken cache. <a href=https://github.com/cortexproject/cortex/pull/3051>#3051</a>, <a href=https://github.com/cortexproject/cortex/pull/3189>#3189</a></li><li>[ENHANCEMENT] <code>-ruler.evaluation-delay-duration</code> is now overridable as a per-tenant limit, <code>ruler_evaluation_delay_duration</code>. <a href=https://github.com/cortexproject/cortex/pull/3098>#3098</a></li><li>[ENHANCEMENT] Add TLS support to etcd client. <a href=https://github.com/cortexproject/cortex/pull/3102>#3102</a></li><li>[ENHANCEMENT] When a tenant accesses the Alertmanager UI or its API, if we have valid <code>-alertmanager.configs.fallback</code> we&rsquo;ll use that to start the manager and avoid failing the request. <a href=https://github.com/cortexproject/cortex/pull/3073>#3073</a></li><li>[ENHANCEMENT] Add <code>DELETE api/v1/rules/{namespace}</code> to the Ruler. It allows all the rule groups of a namespace to be deleted. <a href=https://github.com/cortexproject/cortex/pull/3120>#3120</a></li><li>[ENHANCEMENT] Experimental Delete Series: Retry processing of Delete requests during failures. <a href=https://github.com/cortexproject/cortex/pull/2926>#2926</a></li><li>[ENHANCEMENT] Improve performance of QueryStream() in ingesters. <a href=https://github.com/cortexproject/cortex/pull/3177>#3177</a></li><li>[ENHANCEMENT] Modules included in &ldquo;All&rdquo; target are now visible in output of <code>-modules</code> CLI flag. <a href=https://github.com/cortexproject/cortex/pull/3155>#3155</a></li><li>[ENHANCEMENT] Added <code>/debug/fgprof</code> endpoint to debug running Cortex process using <code>fgprof</code>. This adds up to the existing <code>/debug/...</code> endpoints. <a href=https://github.com/cortexproject/cortex/pull/3131>#3131</a></li><li>[ENHANCEMENT] Blocks storage: optimised <code>/api/v1/series</code> for blocks storage. (<a href=https://github.com/cortexproject/cortex/pull/2976>#2976</a>)</li><li>[BUGFIX] Ruler: when loading rules from &ldquo;local&rdquo; storage, check for directory after resolving symlink. <a href=https://github.com/cortexproject/cortex/pull/3137>#3137</a></li><li>[BUGFIX] Query-frontend: Fixed rounding for incoming query timestamps, to be 100% Prometheus compatible. <a href=https://github.com/cortexproject/cortex/pull/2990>#2990</a></li><li>[BUGFIX] Querier: Merge results from chunks and blocks ingesters when using streaming of results. <a href=https://github.com/cortexproject/cortex/pull/3013>#3013</a></li><li>[BUGFIX] Querier: query /series from ingesters regardless the <code>-querier.query-ingesters-within</code> setting. <a href=https://github.com/cortexproject/cortex/pull/3035>#3035</a></li><li>[BUGFIX] Blocks storage: Ingester is less likely to hit gRPC message size limit when streaming data to queriers. <a href=https://github.com/cortexproject/cortex/pull/3015>#3015</a></li><li>[BUGFIX] Blocks storage: fixed memberlist support for the store-gateways and compactors ring used when blocks sharding is enabled. <a href=https://github.com/cortexproject/cortex/pull/3058>#3058</a> <a href=https://github.com/cortexproject/cortex/pull/3095>#3095</a></li><li>[BUGFIX] Fix configuration for TLS server validation, TLS skip verify was hardcoded to true for all TLS configurations and prevented validation of server certificates. <a href=https://github.com/cortexproject/cortex/pull/3030>#3030</a></li><li>[BUGFIX] Fixes the Alertmanager panicking when no <code>-alertmanager.web.external-url</code> is provided. <a href=https://github.com/cortexproject/cortex/pull/3017>#3017</a></li><li>[BUGFIX] Fixes the registration of the Alertmanager API metrics <code>cortex_alertmanager_alerts_received_total</code> and <code>cortex_alertmanager_alerts_invalid_total</code>. <a href=https://github.com/cortexproject/cortex/pull/3065>#3065</a></li><li>[BUGFIX] Fixes <code>flag needs an argument: -config.expand-env</code> error. <a href=https://github.com/cortexproject/cortex/pull/3087>#3087</a></li><li>[BUGFIX] An index optimisation actually slows things down when using caching. Moved it to the right location. <a href=https://github.com/cortexproject/cortex/pull/2973>#2973</a></li><li>[BUGFIX] Ingester: If push request contained both valid and invalid samples, valid samples were ingested but not stored to WAL of the chunks storage. This has been fixed. <a href=https://github.com/cortexproject/cortex/pull/3067>#3067</a></li><li>[BUGFIX] Cassandra: fixed consistency setting in the CQL session when creating the keyspace. <a href=https://github.com/cortexproject/cortex/pull/3105>#3105</a></li><li>[BUGFIX] Ruler: Config API would return both the <code>record</code> and <code>alert</code> in <code>YAML</code> response keys even when one of them must be empty. <a href=https://github.com/cortexproject/cortex/pull/3120>#3120</a></li><li>[BUGFIX] Index page now uses configured HTTP path prefix when creating links. <a href=https://github.com/cortexproject/cortex/pull/3126>#3126</a></li><li>[BUGFIX] Purger: fixed deadlock when reloading of tombstones failed. <a href=https://github.com/cortexproject/cortex/pull/3182>#3182</a></li><li>[BUGFIX] Fixed panic in flusher job, when error writing chunks to the store would cause &ldquo;idle&rdquo; chunks to be flushed, which triggered panic. <a href=https://github.com/cortexproject/cortex/pull/3140>#3140</a></li><li>[BUGFIX] Index page no longer shows links that are not valid for running Cortex instance. <a href=https://github.com/cortexproject/cortex/pull/3133>#3133</a></li><li>[BUGFIX] Configs: prevent validation of templates to fail when using template functions. <a href=https://github.com/cortexproject/cortex/pull/3157>#3157</a></li><li>[BUGFIX] Configuring the S3 URL with an <code>@</code> but without username and password doesn&rsquo;t enable the AWS static credentials anymore. <a href=https://github.com/cortexproject/cortex/pull/3170>#3170</a></li><li>[BUGFIX] Limit errors on ranged queries (<code>api/v1/query_range</code>) no longer return a status code <code>500</code> but <code>422</code> instead. <a href=https://github.com/cortexproject/cortex/pull/3167>#3167</a></li><li>[BUGFIX] Handle hash-collisions in the query path. Before this fix, Cortex could occasionally mix up two different series in a query, leading to invalid results, when <code>-querier.ingester-streaming</code> was used. <a href=https://github.com/cortexproject/cortex/pull/3192>#3192</a></li></ul><h2 id=130--2020-08-21>1.3.0 / 2020-08-21</h2><ul><li>[CHANGE] Replace the metric <code>cortex_alertmanager_configs</code> with <code>cortex_alertmanager_config_invalid</code> exposed by Alertmanager. <a href=https://github.com/cortexproject/cortex/pull/2960>#2960</a></li><li>[CHANGE] Experimental Delete Series: Change target flag for purger from <code>data-purger</code> to <code>purger</code>. <a href=https://github.com/cortexproject/cortex/pull/2777>#2777</a></li><li>[CHANGE] Experimental blocks storage: The max concurrent queries against the long-term storage, configured via <code>-experimental.blocks-storage.bucket-store.max-concurrent</code>, is now a limit shared across all tenants and not a per-tenant limit anymore. The default value has changed from <code>20</code> to <code>100</code> and the following new metrics have been added: <a href=https://github.com/cortexproject/cortex/pull/2797>#2797</a><ul><li><code>cortex_bucket_stores_gate_queries_concurrent_max</code></li><li><code>cortex_bucket_stores_gate_queries_in_flight</code></li><li><code>cortex_bucket_stores_gate_duration_seconds</code></li></ul></li><li>[CHANGE] Metric <code>cortex_ingester_flush_reasons</code> has been renamed to <code>cortex_ingester_flushing_enqueued_series_total</code>, and new metric <code>cortex_ingester_flushing_dequeued_series_total</code> with <code>outcome</code> label (superset of reason) has been added. <a href=https://github.com/cortexproject/cortex/pull/2802>#2802</a> <a href=https://github.com/cortexproject/cortex/pull/2818>#2818</a> <a href=https://github.com/cortexproject/cortex/pull/2998>#2998</a></li><li>[CHANGE] Experimental Delete Series: Metric <code>cortex_purger_oldest_pending_delete_request_age_seconds</code> would track age of delete requests since they are over their cancellation period instead of their creation time. <a href=https://github.com/cortexproject/cortex/pull/2806>#2806</a></li><li>[CHANGE] Experimental blocks storage: the store-gateway service is required in a Cortex cluster running with the experimental blocks storage. Removed the <code>-experimental.tsdb.store-gateway-enabled</code> CLI flag and <code>store_gateway_enabled</code> YAML config option. The store-gateway is now always enabled when the storage engine is <code>blocks</code>. <a href=https://github.com/cortexproject/cortex/pull/2822>#2822</a></li><li>[CHANGE] Experimental blocks storage: removed support for <code>-experimental.blocks-storage.bucket-store.max-sample-count</code> flag because the implementation was flawed. To limit the number of samples/chunks processed by a single query you can set <code>-store.query-chunk-limit</code>, which is now supported by the blocks storage too. <a href=https://github.com/cortexproject/cortex/pull/2852>#2852</a></li><li>[CHANGE] Ingester: Chunks flushed via /flush stay in memory until retention period is reached. This affects <code>cortex_ingester_memory_chunks</code> metric. <a href=https://github.com/cortexproject/cortex/pull/2778>#2778</a></li><li>[CHANGE] Querier: the error message returned when the query time range exceeds <code>-store.max-query-length</code> has changed from <code>invalid query, length > limit (X > Y)</code> to <code>the query time range exceeds the limit (query length: X, limit: Y)</code>. <a href=https://github.com/cortexproject/cortex/pull/2826>#2826</a></li><li>[CHANGE] Add <code>component</code> label to metrics exposed by chunk, delete and index store clients. <a href=https://github.com/cortexproject/cortex/pull/2774>#2774</a></li><li>[CHANGE] Querier: when <code>-querier.query-ingesters-within</code> is configured, the time range of the query sent to ingesters is now manipulated to ensure the query start time is not older than &rsquo;now - query-ingesters-within&rsquo;. <a href=https://github.com/cortexproject/cortex/pull/2904>#2904</a></li><li>[CHANGE] KV: The <code>role</code> label which was a label of <code>multi</code> KV store client only has been added to metrics of every KV store client. If KV store client is not <code>multi</code>, then the value of <code>role</code> label is <code>primary</code>. <a href=https://github.com/cortexproject/cortex/pull/2837>#2837</a></li><li>[CHANGE] Added the <code>engine</code> label to the metrics exposed by the Prometheus query engine, to distinguish between <code>ruler</code> and <code>querier</code> metrics. <a href=https://github.com/cortexproject/cortex/pull/2854>#2854</a></li><li>[CHANGE] Added ruler to the single binary when started with <code>-target=all</code> (default). <a href=https://github.com/cortexproject/cortex/pull/2854>#2854</a></li><li>[CHANGE] Experimental blocks storage: compact head when opening TSDB. This should only affect ingester startup after it was unable to compact head in previous run. <a href=https://github.com/cortexproject/cortex/pull/2870>#2870</a></li><li>[CHANGE] Metric <code>cortex_overrides_last_reload_successful</code> has been renamed to <code>cortex_runtime_config_last_reload_successful</code>. <a href=https://github.com/cortexproject/cortex/pull/2874>#2874</a></li><li>[CHANGE] HipChat support has been removed from the alertmanager (because removed from the Prometheus upstream too). <a href=https://github.com/cortexproject/cortex/pull/2902>#2902</a></li><li>[CHANGE] Add constant label <code>name</code> to metric <code>cortex_cache_request_duration_seconds</code>. <a href=https://github.com/cortexproject/cortex/pull/2903>#2903</a></li><li>[CHANGE] Add <code>user</code> label to metric <code>cortex_query_frontend_queue_length</code>. <a href=https://github.com/cortexproject/cortex/pull/2939>#2939</a></li><li>[CHANGE] Experimental blocks storage: cleaned up the config and renamed &ldquo;TSDB&rdquo; to &ldquo;blocks storage&rdquo;. <a href=https://github.com/cortexproject/cortex/pull/2937>#2937</a><ul><li>The storage engine setting value has been changed from <code>tsdb</code> to <code>blocks</code>; this affects <code>-store.engine</code> CLI flag and its respective YAML option.</li><li>The root level YAML config has changed from <code>tsdb</code> to <code>blocks_storage</code></li><li>The prefix of all CLI flags has changed from <code>-experimental.tsdb.</code> to <code>-experimental.blocks-storage.</code></li><li>The following settings have been grouped under <code>tsdb</code> property in the YAML config and their CLI flags changed:<ul><li><code>-experimental.tsdb.dir</code> changed to <code>-experimental.blocks-storage.tsdb.dir</code></li><li><code>-experimental.tsdb.block-ranges-period</code> changed to <code>-experimental.blocks-storage.tsdb.block-ranges-period</code></li><li><code>-experimental.tsdb.retention-period</code> changed to <code>-experimental.blocks-storage.tsdb.retention-period</code></li><li><code>-experimental.tsdb.ship-interval</code> changed to <code>-experimental.blocks-storage.tsdb.ship-interval</code></li><li><code>-experimental.tsdb.ship-concurrency</code> changed to <code>-experimental.blocks-storage.tsdb.ship-concurrency</code></li><li><code>-experimental.tsdb.max-tsdb-opening-concurrency-on-startup</code> changed to <code>-experimental.blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup</code></li><li><code>-experimental.tsdb.head-compaction-interval</code> changed to <code>-experimental.blocks-storage.tsdb.head-compaction-interval</code></li><li><code>-experimental.tsdb.head-compaction-concurrency</code> changed to <code>-experimental.blocks-storage.tsdb.head-compaction-concurrency</code></li><li><code>-experimental.tsdb.head-compaction-idle-timeout</code> changed to <code>-experimental.blocks-storage.tsdb.head-compaction-idle-timeout</code></li><li><code>-experimental.tsdb.stripe-size</code> changed to <code>-experimental.blocks-storage.tsdb.stripe-size</code></li><li><code>-experimental.tsdb.wal-compression-enabled</code> changed to <code>-experimental.blocks-storage.tsdb.wal-compression-enabled</code></li><li><code>-experimental.tsdb.flush-blocks-on-shutdown</code> changed to <code>-experimental.blocks-storage.tsdb.flush-blocks-on-shutdown</code></li></ul></li></ul></li><li>[CHANGE] Flags <code>-bigtable.grpc-use-gzip-compression</code>, <code>-ingester.client.grpc-use-gzip-compression</code>, <code>-querier.frontend-client.grpc-use-gzip-compression</code> are now deprecated. <a href=https://github.com/cortexproject/cortex/pull/2940>#2940</a></li><li>[CHANGE] Limit errors reported by ingester during query-time now return HTTP status code 422. <a href=https://github.com/cortexproject/cortex/pull/2941>#2941</a></li><li>[FEATURE] Introduced <code>ruler.for-outage-tolerance</code>, Max time to tolerate outage for restoring &ldquo;for&rdquo; state of alert. <a href=https://github.com/cortexproject/cortex/pull/2783>#2783</a></li><li>[FEATURE] Introduced <code>ruler.for-grace-period</code>, Minimum duration between alert and restored &ldquo;for&rdquo; state. This is maintained only for alerts with configured &ldquo;for&rdquo; time greater than grace period. <a href=https://github.com/cortexproject/cortex/pull/2783>#2783</a></li><li>[FEATURE] Introduced <code>ruler.resend-delay</code>, Minimum amount of time to wait before resending an alert to Alertmanager. <a href=https://github.com/cortexproject/cortex/pull/2783>#2783</a></li><li>[FEATURE] Ruler: added <code>local</code> filesystem support to store rules (read-only). <a href=https://github.com/cortexproject/cortex/pull/2854>#2854</a></li><li>[ENHANCEMENT] Upgraded Docker base images to <code>alpine:3.12</code>. <a href=https://github.com/cortexproject/cortex/pull/2862>#2862</a></li><li>[ENHANCEMENT] Experimental: Querier can now optionally query secondary store. This is specified by using <code>-querier.second-store-engine</code> option, with values <code>chunks</code> or <code>blocks</code>. Standard configuration options for this store are used. Additionally, this querying can be configured to happen only for queries that need data older than <code>-querier.use-second-store-before-time</code>. Default value of zero will always query secondary store. <a href=https://github.com/cortexproject/cortex/pull/2747>#2747</a></li><li>[ENHANCEMENT] Query-tee: increased the <code>cortex_querytee_request_duration_seconds</code> metric buckets granularity. <a href=https://github.com/cortexproject/cortex/pull/2799>#2799</a></li><li>[ENHANCEMENT] Query-tee: fail to start if the configured <code>-backend.preferred</code> is unknown. <a href=https://github.com/cortexproject/cortex/pull/2799>#2799</a></li><li>[ENHANCEMENT] Ruler: Added the following metrics: <a href=https://github.com/cortexproject/cortex/pull/2786>#2786</a><ul><li><code>cortex_prometheus_notifications_latency_seconds</code></li><li><code>cortex_prometheus_notifications_errors_total</code></li><li><code>cortex_prometheus_notifications_sent_total</code></li><li><code>cortex_prometheus_notifications_dropped_total</code></li><li><code>cortex_prometheus_notifications_queue_length</code></li><li><code>cortex_prometheus_notifications_queue_capacity</code></li><li><code>cortex_prometheus_notifications_alertmanagers_discovered</code></li></ul></li><li>[ENHANCEMENT] The behavior of the <code>/ready</code> was changed for the query frontend to indicate when it was ready to accept queries. This is intended for use by a read path load balancer that would want to wait for the frontend to have attached queriers before including it in the backend. <a href=https://github.com/cortexproject/cortex/pull/2733>#2733</a></li><li>[ENHANCEMENT] Experimental Delete Series: Add support for deletion of chunks for remaining stores. <a href=https://github.com/cortexproject/cortex/pull/2801>#2801</a></li><li>[ENHANCEMENT] Add <code>-modules</code> command line flag to list possible values for <code>-target</code>. Also, log warning if given target is internal component. <a href=https://github.com/cortexproject/cortex/pull/2752>#2752</a></li><li>[ENHANCEMENT] Added <code>-ingester.flush-on-shutdown-with-wal-enabled</code> option to enable chunks flushing even when WAL is enabled. <a href=https://github.com/cortexproject/cortex/pull/2780>#2780</a></li><li>[ENHANCEMENT] Query-tee: Support for custom API prefix by using <code>-server.path-prefix</code> option. <a href=https://github.com/cortexproject/cortex/pull/2814>#2814</a></li><li>[ENHANCEMENT] Query-tee: Forward <code>X-Scope-OrgId</code> header to backend, if present in the request. <a href=https://github.com/cortexproject/cortex/pull/2815>#2815</a></li><li>[ENHANCEMENT] Experimental blocks storage: Added <code>-experimental.blocks-storage.tsdb.head-compaction-idle-timeout</code> option to force compaction of data in memory into a block. <a href=https://github.com/cortexproject/cortex/pull/2803>#2803</a></li><li>[ENHANCEMENT] Experimental blocks storage: Added support for flushing blocks via <code>/flush</code>, <code>/shutdown</code> (previously these only worked for chunks storage) and by using <code>-experimental.blocks-storage.tsdb.flush-blocks-on-shutdown</code> option. <a href=https://github.com/cortexproject/cortex/pull/2794>#2794</a></li><li>[ENHANCEMENT] Experimental blocks storage: Added support to enforce max query time range length via <code>-store.max-query-length</code>. <a href=https://github.com/cortexproject/cortex/pull/2826>#2826</a></li><li>[ENHANCEMENT] Experimental blocks storage: Added support to limit the max number of chunks that can be fetched from the long-term storage while executing a query. The limit is enforced both in the querier and store-gateway, and is configurable via <code>-store.query-chunk-limit</code>. <a href=https://github.com/cortexproject/cortex/pull/2852>#2852</a> <a href=https://github.com/cortexproject/cortex/pull/2922>#2922</a></li><li>[ENHANCEMENT] Ingester: Added new metric <code>cortex_ingester_flush_series_in_progress</code> that reports number of ongoing flush-series operations. Useful when calling <code>/flush</code> handler: if <code>cortex_ingester_flush_queue_length + cortex_ingester_flush_series_in_progress</code> is 0, all flushes are finished. <a href=https://github.com/cortexproject/cortex/pull/2778>#2778</a></li><li>[ENHANCEMENT] Memberlist members can join cluster via SRV records. <a href=https://github.com/cortexproject/cortex/pull/2788>#2788</a></li><li>[ENHANCEMENT] Added configuration options for chunks s3 client. <a href=https://github.com/cortexproject/cortex/pull/2831>#2831</a><ul><li><code>s3.endpoint</code></li><li><code>s3.region</code></li><li><code>s3.access-key-id</code></li><li><code>s3.secret-access-key</code></li><li><code>s3.insecure</code></li><li><code>s3.sse-encryption</code></li><li><code>s3.http.idle-conn-timeout</code></li><li><code>s3.http.response-header-timeout</code></li><li><code>s3.http.insecure-skip-verify</code></li></ul></li><li>[ENHANCEMENT] Prometheus upgraded. <a href=https://github.com/cortexproject/cortex/pull/2798>#2798</a> <a href=https://github.com/cortexproject/cortex/pull/2849>#2849</a> <a href=https://github.com/cortexproject/cortex/pull/2867>#2867</a> <a href=https://github.com/cortexproject/cortex/pull/2902>#2902</a> <a href=https://github.com/cortexproject/cortex/pull/2918>#2918</a><ul><li>Optimized labels regex matchers for patterns containing literals (eg. <code>foo.*</code>, <code>.*foo</code>, <code>.*foo.*</code>)</li></ul></li><li>[ENHANCEMENT] Add metric <code>cortex_ruler_config_update_failures_total</code> to Ruler to track failures of loading rules files. <a href=https://github.com/cortexproject/cortex/pull/2857>#2857</a></li><li>[ENHANCEMENT] Experimental Alertmanager: Alertmanager configuration persisted to object storage using an experimental API that accepts and returns YAML-based Alertmanager configuration. <a href=https://github.com/cortexproject/cortex/pull/2768>#2768</a></li><li>[ENHANCEMENT] Ruler: <code>-ruler.alertmanager-url</code> now supports multiple URLs. Each URL is treated as a separate Alertmanager group. Support for multiple Alertmanagers in a group can be achieved by using DNS service discovery. <a href=https://github.com/cortexproject/cortex/pull/2851>#2851</a></li><li>[ENHANCEMENT] Experimental blocks storage: Cortex Flusher now works with blocks engine. Flusher needs to be provided with blocks-engine configuration, existing Flusher flags are not used (they are only relevant for chunks engine). Note that flush errors are only reported via log. <a href=https://github.com/cortexproject/cortex/pull/2877>#2877</a></li><li>[ENHANCEMENT] Flusher: Added <code>-flusher.exit-after-flush</code> option (defaults to true) to control whether Cortex should stop completely after Flusher has finished its work. <a href=https://github.com/cortexproject/cortex/pull/2877>#2877</a></li><li>[ENHANCEMENT] Added metrics <code>cortex_config_hash</code> and <code>cortex_runtime_config_hash</code> to expose hash of the currently active config file. <a href=https://github.com/cortexproject/cortex/pull/2874>#2874</a></li><li>[ENHANCEMENT] Logger: added JSON logging support, configured via the <code>-log.format=json</code> CLI flag or its respective YAML config option. <a href=https://github.com/cortexproject/cortex/pull/2386>#2386</a></li><li>[ENHANCEMENT] Added new flags <code>-bigtable.grpc-compression</code>, <code>-ingester.client.grpc-compression</code>, <code>-querier.frontend-client.grpc-compression</code> to configure compression used by gRPC. Valid values are <code>gzip</code>, <code>snappy</code>, or empty string (no compression, default). <a href=https://github.com/cortexproject/cortex/pull/2940>#2940</a></li><li>[ENHANCEMENT] Clarify limitations of the <code>/api/v1/series</code>, <code>/api/v1/labels</code> and <code>/api/v1/label/{name}/values</code> endpoints. <a href=https://github.com/cortexproject/cortex/pull/2953>#2953</a></li><li>[ENHANCEMENT] Ingester: added <code>Dropped</code> outcome to metric <code>cortex_ingester_flushing_dequeued_series_total</code>. <a href=https://github.com/cortexproject/cortex/pull/2998>#2998</a></li><li>[BUGFIX] Fixed a bug with <code>api/v1/query_range</code> where no responses would return null values for <code>result</code> and empty values for <code>resultType</code>. <a href=https://github.com/cortexproject/cortex/pull/2962>#2962</a></li><li>[BUGFIX] Fixed a bug in the index intersect code causing storage to return more chunks/series than required. <a href=https://github.com/cortexproject/cortex/pull/2796>#2796</a></li><li>[BUGFIX] Fixed the number of reported keys in the background cache queue. <a href=https://github.com/cortexproject/cortex/pull/2764>#2764</a></li><li>[BUGFIX] Fix race in processing of headers in sharded queries. <a href=https://github.com/cortexproject/cortex/pull/2762>#2762</a></li><li>[BUGFIX] Query Frontend: Do not re-split sharded requests around ingester boundaries. <a href=https://github.com/cortexproject/cortex/pull/2766>#2766</a></li><li>[BUGFIX] Experimental Delete Series: Fixed a problem with cache generation numbers prefixed to cache keys. <a href=https://github.com/cortexproject/cortex/pull/2800>#2800</a></li><li>[BUGFIX] Ingester: Flushing chunks via <code>/flush</code> endpoint could previously lead to panic, if chunks were already flushed before and then removed from memory during the flush caused by <code>/flush</code> handler. Immediate flush now doesn&rsquo;t cause chunks to be flushed again. Samples received during flush triggered via <code>/flush</code> handler are no longer discarded. <a href=https://github.com/cortexproject/cortex/pull/2778>#2778</a></li><li>[BUGFIX] Prometheus upgraded. <a href=https://github.com/cortexproject/cortex/pull/2849>#2849</a><ul><li>Fixed unknown symbol error during head compaction</li></ul></li><li>[BUGFIX] Fix panic when using cassandra as store for both index and delete requests. <a href=https://github.com/cortexproject/cortex/pull/2774>#2774</a></li><li>[BUGFIX] Experimental Delete Series: Fixed a data race in Purger. <a href=https://github.com/cortexproject/cortex/pull/2817>#2817</a></li><li>[BUGFIX] KV: Fixed a bug that triggered a panic due to metrics being registered with the same name but different labels when using a <code>multi</code> configured KV client. <a href=https://github.com/cortexproject/cortex/pull/2837>#2837</a></li><li>[BUGFIX] Query-frontend: Fix passing HTTP <code>Host</code> header if <code>-frontend.downstream-url</code> is configured. <a href=https://github.com/cortexproject/cortex/pull/2880>#2880</a></li><li>[BUGFIX] Ingester: Improve time-series distribution when <code>-experimental.distributor.user-subring-size</code> is enabled. <a href=https://github.com/cortexproject/cortex/pull/2887>#2887</a></li><li>[BUGFIX] Set content type to <code>application/x-protobuf</code> for remote_read responses. <a href=https://github.com/cortexproject/cortex/pull/2915>#2915</a></li><li>[BUGFIX] Fixed ruler and store-gateway instance registration in the ring (when sharding is enabled) when a new instance replaces abruptly terminated one, and the only difference between the two instances is the address. <a href=https://github.com/cortexproject/cortex/pull/2954>#2954</a></li><li>[BUGFIX] Fixed <code>Missing chunks and index config causing silent failure</code> Absence of chunks and index from schema config is not validated. <a href=https://github.com/cortexproject/cortex/pull/2732>#2732</a></li><li>[BUGFIX] Fix panic caused by KVs from boltdb being used beyond their life. <a href=https://github.com/cortexproject/cortex/pull/2971>#2971</a></li><li>[BUGFIX] Experimental blocks storage: <code>/api/v1/series</code>, <code>/api/v1/labels</code> and <code>/api/v1/label/{name}/values</code> only query the TSDB head regardless of the configured <code>-experimental.blocks-storage.tsdb.retention-period</code>. <a href=https://github.com/cortexproject/cortex/pull/2974>#2974</a></li><li>[BUGFIX] Ingester: Avoid indefinite checkpointing in case of surge in number of series. <a href=https://github.com/cortexproject/cortex/pull/2955>#2955</a></li><li>[BUGFIX] Querier: query /series from ingesters regardless the <code>-querier.query-ingesters-within</code> setting. <a href=https://github.com/cortexproject/cortex/pull/3035>#3035</a></li><li>[BUGFIX] Ruler: fixed an unintentional breaking change introduced in the ruler&rsquo;s <code>alertmanager_url</code> YAML config option, which changed the value from a string to a list of strings. <a href=https://github.com/cortexproject/cortex/pull/2989>#2989</a></li></ul><h2 id=120--2020-07-01>1.2.0 / 2020-07-01</h2><ul><li>[CHANGE] Metric <code>cortex_kv_request_duration_seconds</code> now includes <code>name</code> label to denote which client is being used as well as the <code>backend</code> label to denote the KV backend implementation in use. <a href=https://github.com/cortexproject/cortex/pull/2648>#2648</a></li><li>[CHANGE] Experimental Ruler: Rule groups persisted to object storage using the experimental API have an updated object key encoding to better handle special characters. Rule groups previously-stored using object storage must be renamed to the new format. <a href=https://github.com/cortexproject/cortex/pull/2646>#2646</a></li><li>[CHANGE] Query Frontend now uses Round Robin to choose a tenant queue to service next. <a href=https://github.com/cortexproject/cortex/pull/2553>#2553</a></li><li>[CHANGE] <code>-promql.lookback-delta</code> is now deprecated and has been replaced by <code>-querier.lookback-delta</code> along with <code>lookback_delta</code> entry under <code>querier</code> in the config file. <code>-promql.lookback-delta</code> will be removed in v1.4.0. <a href=https://github.com/cortexproject/cortex/pull/2604>#2604</a></li><li>[CHANGE] Experimental TSDB: removed <code>-experimental.tsdb.bucket-store.binary-index-header-enabled</code> flag. Now the binary index-header is always enabled.</li><li>[CHANGE] Experimental TSDB: Renamed index-cache metrics to use original metric names from Thanos, as Cortex is not aggregating them in any way: <a href=https://github.com/cortexproject/cortex/pull/2627>#2627</a><ul><li><code>cortex_&lt;service>_blocks_index_cache_items_evicted_total</code> => <code>thanos_store_index_cache_items_evicted_total{name="index-cache"}</code></li><li><code>cortex_&lt;service>_blocks_index_cache_items_added_total</code> => <code>thanos_store_index_cache_items_added_total{name="index-cache"}</code></li><li><code>cortex_&lt;service>_blocks_index_cache_requests_total</code> => <code>thanos_store_index_cache_requests_total{name="index-cache"}</code></li><li><code>cortex_&lt;service>_blocks_index_cache_items_overflowed_total</code> => <code>thanos_store_index_cache_items_overflowed_total{name="index-cache"}</code></li><li><code>cortex_&lt;service>_blocks_index_cache_hits_total</code> => <code>thanos_store_index_cache_hits_total{name="index-cache"}</code></li><li><code>cortex_&lt;service>_blocks_index_cache_items</code> => <code>thanos_store_index_cache_items{name="index-cache"}</code></li><li><code>cortex_&lt;service>_blocks_index_cache_items_size_bytes</code> => <code>thanos_store_index_cache_items_size_bytes{name="index-cache"}</code></li><li><code>cortex_&lt;service>_blocks_index_cache_total_size_bytes</code> => <code>thanos_store_index_cache_total_size_bytes{name="index-cache"}</code></li><li><code>cortex_&lt;service>_blocks_index_cache_memcached_operations_total</code> => <code>thanos_memcached_operations_total{name="index-cache"}</code></li><li><code>cortex_&lt;service>_blocks_index_cache_memcached_operation_failures_total</code> => <code>thanos_memcached_operation_failures_total{name="index-cache"}</code></li><li><code>cortex_&lt;service>_blocks_index_cache_memcached_operation_duration_seconds</code> => <code>thanos_memcached_operation_duration_seconds{name="index-cache"}</code></li><li><code>cortex_&lt;service>_blocks_index_cache_memcached_operation_skipped_total</code> => <code>thanos_memcached_operation_skipped_total{name="index-cache"}</code></li></ul></li><li>[CHANGE] Experimental TSDB: Renamed metrics in bucket stores: <a href=https://github.com/cortexproject/cortex/pull/2627>#2627</a><ul><li><code>cortex_&lt;service>_blocks_meta_syncs_total</code> => <code>cortex_blocks_meta_syncs_total{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_blocks_meta_sync_failures_total</code> => <code>cortex_blocks_meta_sync_failures_total{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_blocks_meta_sync_duration_seconds</code> => <code>cortex_blocks_meta_sync_duration_seconds{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_blocks_meta_sync_consistency_delay_seconds</code> => <code>cortex_blocks_meta_sync_consistency_delay_seconds{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_blocks_meta_synced</code> => <code>cortex_blocks_meta_synced{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_block_loads_total</code> => <code>cortex_bucket_store_block_loads_total{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_block_load_failures_total</code> => <code>cortex_bucket_store_block_load_failures_total{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_block_drops_total</code> => <code>cortex_bucket_store_block_drops_total{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_block_drop_failures_total</code> => <code>cortex_bucket_store_block_drop_failures_total{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_blocks_loaded</code> => <code>cortex_bucket_store_blocks_loaded{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_series_data_touched</code> => <code>cortex_bucket_store_series_data_touched{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_series_data_fetched</code> => <code>cortex_bucket_store_series_data_fetched{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_series_data_size_touched_bytes</code> => <code>cortex_bucket_store_series_data_size_touched_bytes{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_series_data_size_fetched_bytes</code> => <code>cortex_bucket_store_series_data_size_fetched_bytes{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_series_blocks_queried</code> => <code>cortex_bucket_store_series_blocks_queried{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_series_get_all_duration_seconds</code> => <code>cortex_bucket_store_series_get_all_duration_seconds{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_series_merge_duration_seconds</code> => <code>cortex_bucket_store_series_merge_duration_seconds{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_series_refetches_total</code> => <code>cortex_bucket_store_series_refetches_total{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_series_result_series</code> => <code>cortex_bucket_store_series_result_series{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_cached_postings_compressions_total</code> => <code>cortex_bucket_store_cached_postings_compressions_total{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_cached_postings_compression_errors_total</code> => <code>cortex_bucket_store_cached_postings_compression_errors_total{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_cached_postings_compression_time_seconds</code> => <code>cortex_bucket_store_cached_postings_compression_time_seconds{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_cached_postings_original_size_bytes_total</code> => <code>cortex_bucket_store_cached_postings_original_size_bytes_total{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_bucket_store_cached_postings_compressed_size_bytes_total</code> => <code>cortex_bucket_store_cached_postings_compressed_size_bytes_total{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_blocks_sync_seconds</code> => <code>cortex_bucket_stores_blocks_sync_seconds{component="&lt;service>"}</code></li><li><code>cortex_&lt;service>_blocks_last_successful_sync_timestamp_seconds</code> => <code>cortex_bucket_stores_blocks_last_successful_sync_timestamp_seconds{component="&lt;service>"}</code></li></ul></li><li>[CHANGE] Available command-line flags are printed to stdout, and only when requested via <code>-help</code>. Using invalid flag no longer causes printing of all available flags. <a href=https://github.com/cortexproject/cortex/pull/2691>#2691</a></li><li>[CHANGE] Experimental Memberlist ring: randomize gossip node names to avoid conflicts when running multiple clients on the same host, or reusing host names (eg. pods in statefulset). Node name randomization can be disabled by using <code>-memberlist.randomize-node-name=false</code>. <a href=https://github.com/cortexproject/cortex/pull/2715>#2715</a></li><li>[CHANGE] Memberlist KV client is no longer considered experimental. <a href=https://github.com/cortexproject/cortex/pull/2725>#2725</a></li><li>[CHANGE] Experimental Delete Series: Make delete request cancellation duration configurable. <a href=https://github.com/cortexproject/cortex/pull/2760>#2760</a></li><li>[CHANGE] Removed <code>-store.fullsize-chunks</code> option which was undocumented and unused (it broke ingester hand-overs). <a href=https://github.com/cortexproject/cortex/pull/2656>#2656</a></li><li>[CHANGE] Query with no metric name that has previously resulted in HTTP status code 500 now returns status code 422 instead. <a href=https://github.com/cortexproject/cortex/pull/2571>#2571</a></li><li>[FEATURE] TLS config options added for GRPC clients in Querier (Query-frontend client & Ingester client), Ruler, Store Gateway, as well as HTTP client in Config store client. <a href=https://github.com/cortexproject/cortex/pull/2502>#2502</a></li><li>[FEATURE] The flag <code>-frontend.max-cache-freshness</code> is now supported within the limits overrides, to specify per-tenant max cache freshness values. The corresponding YAML config parameter has been changed from <code>results_cache.max_freshness</code> to <code>limits_config.max_cache_freshness</code>. The legacy YAML config parameter (<code>results_cache.max_freshness</code>) will continue to be supported till Cortex release <code>v1.4.0</code>. <a href=https://github.com/cortexproject/cortex/pull/2609>#2609</a></li><li>[FEATURE] Experimental gRPC Store: Added support to 3rd parties index and chunk stores using gRPC client/server plugin mechanism. <a href=https://github.com/cortexproject/cortex/pull/2220>#2220</a></li><li>[FEATURE] Add <code>-cassandra.table-options</code> flag to customize table options of Cassandra when creating the index or chunk table. <a href=https://github.com/cortexproject/cortex/pull/2575>#2575</a></li><li>[ENHANCEMENT] Propagate GOPROXY value when building <code>build-image</code>. This is to help the builders building the code in a Network where default Go proxy is not accessible (e.g. when behind some corporate VPN). <a href=https://github.com/cortexproject/cortex/pull/2741>#2741</a></li><li>[ENHANCEMENT] Querier: Added metric <code>cortex_querier_request_duration_seconds</code> for all requests to the querier. <a href=https://github.com/cortexproject/cortex/pull/2708>#2708</a></li><li>[ENHANCEMENT] Cortex is now built with Go 1.14. <a href=https://github.com/cortexproject/cortex/pull/2480>#2480</a> <a href=https://github.com/cortexproject/cortex/pull/2749>#2749</a> <a href=https://github.com/cortexproject/cortex/pull/2753>#2753</a></li><li>[ENHANCEMENT] Experimental TSDB: added the following metrics to the ingester: <a href=https://github.com/cortexproject/cortex/pull/2580>#2580</a> <a href=https://github.com/cortexproject/cortex/pull/2583>#2583</a> <a href=https://github.com/cortexproject/cortex/pull/2589>#2589</a> <a href=https://github.com/cortexproject/cortex/pull/2654>#2654</a><ul><li><code>cortex_ingester_tsdb_appender_add_duration_seconds</code></li><li><code>cortex_ingester_tsdb_appender_commit_duration_seconds</code></li><li><code>cortex_ingester_tsdb_refcache_purge_duration_seconds</code></li><li><code>cortex_ingester_tsdb_compactions_total</code></li><li><code>cortex_ingester_tsdb_compaction_duration_seconds</code></li><li><code>cortex_ingester_tsdb_wal_fsync_duration_seconds</code></li><li><code>cortex_ingester_tsdb_wal_page_flushes_total</code></li><li><code>cortex_ingester_tsdb_wal_completed_pages_total</code></li><li><code>cortex_ingester_tsdb_wal_truncations_failed_total</code></li><li><code>cortex_ingester_tsdb_wal_truncations_total</code></li><li><code>cortex_ingester_tsdb_wal_writes_failed_total</code></li><li><code>cortex_ingester_tsdb_checkpoint_deletions_failed_total</code></li><li><code>cortex_ingester_tsdb_checkpoint_deletions_total</code></li><li><code>cortex_ingester_tsdb_checkpoint_creations_failed_total</code></li><li><code>cortex_ingester_tsdb_checkpoint_creations_total</code></li><li><code>cortex_ingester_tsdb_wal_truncate_duration_seconds</code></li><li><code>cortex_ingester_tsdb_head_active_appenders</code></li><li><code>cortex_ingester_tsdb_head_series_not_found_total</code></li><li><code>cortex_ingester_tsdb_head_chunks</code></li><li><code>cortex_ingester_tsdb_mmap_chunk_corruptions_total</code></li><li><code>cortex_ingester_tsdb_head_chunks_created_total</code></li><li><code>cortex_ingester_tsdb_head_chunks_removed_total</code></li></ul></li><li>[ENHANCEMENT] Experimental TSDB: added metrics useful to alert on critical conditions of the blocks storage: <a href=https://github.com/cortexproject/cortex/pull/2573>#2573</a><ul><li><code>cortex_compactor_last_successful_run_timestamp_seconds</code></li><li><code>cortex_querier_blocks_last_successful_sync_timestamp_seconds</code> (when store-gateway is disabled)</li><li><code>cortex_querier_blocks_last_successful_scan_timestamp_seconds</code> (when store-gateway is enabled)</li><li><code>cortex_storegateway_blocks_last_successful_sync_timestamp_seconds</code></li></ul></li><li>[ENHANCEMENT] Experimental TSDB: added the flag <code>-experimental.tsdb.wal-compression-enabled</code> to allow to enable TSDB WAL compression. <a href=https://github.com/cortexproject/cortex/pull/2585>#2585</a></li><li>[ENHANCEMENT] Experimental TSDB: Querier and store-gateway components can now use so-called &ldquo;caching bucket&rdquo;, which can currently cache fetched chunks into shared memcached server. <a href=https://github.com/cortexproject/cortex/pull/2572>#2572</a></li><li>[ENHANCEMENT] Ruler: Automatically remove unhealthy rulers from the ring. <a href=https://github.com/cortexproject/cortex/pull/2587>#2587</a></li><li>[ENHANCEMENT] Query-tee: added support to <code>/metadata</code>, <code>/alerts</code>, and <code>/rules</code> endpoints <a href=https://github.com/cortexproject/cortex/pull/2600>#2600</a></li><li>[ENHANCEMENT] Query-tee: added support to query results comparison between two different backends. The comparison is disabled by default and can be enabled via <code>-proxy.compare-responses=true</code>. <a href=https://github.com/cortexproject/cortex/pull/2611>#2611</a></li><li>[ENHANCEMENT] Query-tee: improved the query-tee to not wait all backend responses before sending back the response to the client. The query-tee now sends back to the client first successful response, while honoring the <code>-backend.preferred</code> option. <a href=https://github.com/cortexproject/cortex/pull/2702>#2702</a></li><li>[ENHANCEMENT] Thanos and Prometheus upgraded. <a href=https://github.com/cortexproject/cortex/pull/2602>#2602</a> <a href=https://github.com/cortexproject/cortex/pull/2604>#2604</a> <a href=https://github.com/cortexproject/cortex/pull/2634>#2634</a> <a href=https://github.com/cortexproject/cortex/pull/2659>#2659</a> <a href=https://github.com/cortexproject/cortex/pull/2686>#2686</a> <a href=https://github.com/cortexproject/cortex/pull/2756>#2756</a><ul><li>TSDB now holds less WAL files after Head Truncation.</li><li>TSDB now does memory-mapping of Head chunks and reduces memory usage.</li></ul></li><li>[ENHANCEMENT] Experimental TSDB: decoupled blocks deletion from blocks compaction in the compactor, so that blocks deletion is not blocked by a busy compactor. The following metrics have been added: <a href=https://github.com/cortexproject/cortex/pull/2623>#2623</a><ul><li><code>cortex_compactor_block_cleanup_started_total</code></li><li><code>cortex_compactor_block_cleanup_completed_total</code></li><li><code>cortex_compactor_block_cleanup_failed_total</code></li><li><code>cortex_compactor_block_cleanup_last_successful_run_timestamp_seconds</code></li></ul></li><li>[ENHANCEMENT] Experimental TSDB: Use shared cache for metadata. This is especially useful when running multiple querier and store-gateway components to reduce number of object store API calls. <a href=https://github.com/cortexproject/cortex/pull/2626>#2626</a> <a href=https://github.com/cortexproject/cortex/pull/2640>#2640</a></li><li>[ENHANCEMENT] Experimental TSDB: when <code>-querier.query-store-after</code> is configured and running the experimental blocks storage, the time range of the query sent to the store is now manipulated to ensure the query end time is not more recent than &rsquo;now - query-store-after&rsquo;. <a href=https://github.com/cortexproject/cortex/pull/2642>#2642</a></li><li>[ENHANCEMENT] Experimental TSDB: small performance improvement in concurrent usage of RefCache, used during samples ingestion. <a href=https://github.com/cortexproject/cortex/pull/2651>#2651</a></li><li>[ENHANCEMENT] The following endpoints now respond appropriately to an <code>Accept</code> header with the value <code>application/json</code> <a href=https://github.com/cortexproject/cortex/pull/2673>#2673</a><ul><li><code>/distributor/all_user_stats</code></li><li><code>/distributor/ha_tracker</code></li><li><code>/ingester/ring</code></li><li><code>/store-gateway/ring</code></li><li><code>/compactor/ring</code></li><li><code>/ruler/ring</code></li><li><code>/services</code></li></ul></li><li>[ENHANCEMENT] Experimental Cassandra backend: Add <code>-cassandra.num-connections</code> to allow increasing the number of TCP connections to each Cassandra server. <a href=https://github.com/cortexproject/cortex/pull/2666>#2666</a></li><li>[ENHANCEMENT] Experimental Cassandra backend: Use separate Cassandra clients and connections for reads and writes. <a href=https://github.com/cortexproject/cortex/pull/2666>#2666</a></li><li>[ENHANCEMENT] Experimental Cassandra backend: Add <code>-cassandra.reconnect-interval</code> to allow specifying the reconnect interval to a Cassandra server that has been marked <code>DOWN</code> by the gocql driver. Also change the default value of the reconnect interval from <code>60s</code> to <code>1s</code>. <a href=https://github.com/cortexproject/cortex/pull/2687>#2687</a></li><li>[ENHANCEMENT] Experimental Cassandra backend: Add option <code>-cassandra.convict-hosts-on-failure=false</code> to not convict host of being down when a request fails. <a href=https://github.com/cortexproject/cortex/pull/2684>#2684</a></li><li>[ENHANCEMENT] Experimental TSDB: Applied a jitter to the period bucket scans in order to better distribute bucket operations over the time and increase the probability of hitting the shared cache (if configured). <a href=https://github.com/cortexproject/cortex/pull/2693>#2693</a></li><li>[ENHANCEMENT] Experimental TSDB: Series limit per user and per metric now work in TSDB blocks. <a href=https://github.com/cortexproject/cortex/pull/2676>#2676</a></li><li>[ENHANCEMENT] Experimental Memberlist: Added ability to periodically rejoin the memberlist cluster. <a href=https://github.com/cortexproject/cortex/pull/2724>#2724</a></li><li>[ENHANCEMENT] Experimental Delete Series: Added the following metrics for monitoring processing of delete requests: <a href=https://github.com/cortexproject/cortex/pull/2730>#2730</a><ul><li><code>cortex_purger_load_pending_requests_attempts_total</code>: Number of attempts that were made to load pending requests with status.</li><li><code>cortex_purger_oldest_pending_delete_request_age_seconds</code>: Age of oldest pending delete request in seconds.</li><li><code>cortex_purger_pending_delete_requests_count</code>: Count of requests which are in process or are ready to be processed.</li></ul></li><li>[ENHANCEMENT] Experimental TSDB: Improved compactor to hard-delete also partial blocks with an deletion mark (even if the deletion mark threshold has not been reached). <a href=https://github.com/cortexproject/cortex/pull/2751>#2751</a></li><li>[ENHANCEMENT] Experimental TSDB: Introduced a consistency check done by the querier to ensure all expected blocks have been queried via the store-gateway. If a block is missing on a store-gateway, the querier retries fetching series from missing blocks up to 3 times. If the consistency check fails once all retries have been exhausted, the query execution fails. The following metrics have been added: <a href=https://github.com/cortexproject/cortex/pull/2593>#2593</a> <a href=https://github.com/cortexproject/cortex/pull/2630>#2630</a> <a href=https://github.com/cortexproject/cortex/pull/2689>#2689</a> <a href=https://github.com/cortexproject/cortex/pull/2695>#2695</a><ul><li><code>cortex_querier_blocks_consistency_checks_total</code></li><li><code>cortex_querier_blocks_consistency_checks_failed_total</code></li><li><code>cortex_querier_storegateway_refetches_per_query</code></li></ul></li><li>[ENHANCEMENT] Delete requests can now be canceled <a href=https://github.com/cortexproject/cortex/pull/2555>#2555</a></li><li>[ENHANCEMENT] Table manager can now provision tables for delete store <a href=https://github.com/cortexproject/cortex/pull/2546>#2546</a></li><li>[BUGFIX] Ruler: Ensure temporary rule files with special characters are properly mapped and cleaned up. <a href=https://github.com/cortexproject/cortex/pull/2506>#2506</a></li><li>[BUGFIX] Fixes <a href=https://github.com/cortexproject/cortex/pull/2411>#2411</a>, Ensure requests are properly routed to the prometheus api embedded in the query if <code>-server.path-prefix</code> is set. <a href=https://github.com/cortexproject/cortex/pull/2372>#2372</a></li><li>[BUGFIX] Experimental TSDB: fixed chunk data corruption when querying back series using the experimental blocks storage. <a href=https://github.com/cortexproject/cortex/pull/2400>#2400</a></li><li>[BUGFIX] Fixed collection of tracing spans from Thanos components used internally. <a href=https://github.com/cortexproject/cortex/pull/2655>#2655</a></li><li>[BUGFIX] Experimental TSDB: fixed memory leak in ingesters. <a href=https://github.com/cortexproject/cortex/pull/2586>#2586</a></li><li>[BUGFIX] QueryFrontend: fixed a situation where HTTP error is ignored and an incorrect status code is set. <a href=https://github.com/cortexproject/cortex/pull/2590>#2590</a></li><li>[BUGFIX] Ingester: Fix an ingester starting up in the JOINING state and staying there forever. <a href=https://github.com/cortexproject/cortex/pull/2565>#2565</a></li><li>[BUGFIX] QueryFrontend: fixed a panic (<code>integer divide by zero</code>) in the query-frontend. The query-frontend now requires the <code>-querier.default-evaluation-interval</code> config to be set to the same value of the querier. <a href=https://github.com/cortexproject/cortex/pull/2614>#2614</a></li><li>[BUGFIX] Experimental TSDB: when the querier receives a <code>/series</code> request with a time range older than the data stored in the ingester, it now ignores the requested time range and returns known series anyway instead of returning an empty response. This aligns the behaviour with the chunks storage. <a href=https://github.com/cortexproject/cortex/pull/2617>#2617</a></li><li>[BUGFIX] Cassandra: fixed an edge case leading to an invalid CQL query when querying the index on a Cassandra store. <a href=https://github.com/cortexproject/cortex/pull/2639>#2639</a></li><li>[BUGFIX] Ingester: increment series per metric when recovering from WAL or transfer. <a href=https://github.com/cortexproject/cortex/pull/2674>#2674</a></li><li>[BUGFIX] Fixed <code>wrong number of arguments for 'mget' command</code> Redis error when a query has no chunks to lookup from storage. <a href=https://github.com/cortexproject/cortex/pull/2700>#2700</a> <a href=https://github.com/cortexproject/cortex/pull/2796>#2796</a></li><li>[BUGFIX] Ingester: Automatically remove old tmp checkpoints, fixing a potential disk space leak after an ingester crashes. <a href=https://github.com/cortexproject/cortex/pull/2726>#2726</a></li></ul><h2 id=110--2020-05-21>1.1.0 / 2020-05-21</h2><p>This release brings the usual mix of bugfixes and improvements. The biggest change is that WAL support for chunks is now considered to be production-ready!</p><p>Please make sure to review renamed metrics, and update your dashboards and alerts accordingly.</p><ul><li>[CHANGE] Added v1 API routes documented in <a href=https://github.com/cortexproject/cortex/pull/2327>#2327</a>. <a href=https://github.com/cortexproject/cortex/pull/2372>#2372</a><ul><li>Added <code>-http.alertmanager-http-prefix</code> flag which allows the configuration of the path where the Alertmanager API and UI can be reached. The default is set to <code>/alertmanager</code>.</li><li>Added <code>-http.prometheus-http-prefix</code> flag which allows the configuration of the path where the Prometheus API and UI can be reached. The default is set to <code>/prometheus</code>.</li><li>Updated the index hosted at the root prefix to point to the updated routes.</li><li>Legacy routes hardcoded with the <code>/api/prom</code> prefix now respect the <code>-http.prefix</code> flag.</li></ul></li><li>[CHANGE] The metrics <code>cortex_distributor_ingester_appends_total</code> and <code>distributor_ingester_append_failures_total</code> now include a <code>type</code> label to differentiate between <code>samples</code> and <code>metadata</code>. <a href=https://github.com/cortexproject/cortex/pull/2336>#2336</a></li><li>[CHANGE] The metrics for number of chunks and bytes flushed to the chunk store are renamed. Note that previous metrics were counted pre-deduplication, while new metrics are counted after deduplication. <a href=https://github.com/cortexproject/cortex/pull/2463>#2463</a><ul><li><code>cortex_ingester_chunks_stored_total</code> > <code>cortex_chunk_store_stored_chunks_total</code></li><li><code>cortex_ingester_chunk_stored_bytes_total</code> > <code>cortex_chunk_store_stored_chunk_bytes_total</code></li></ul></li><li>[CHANGE] Experimental TSDB: renamed blocks meta fetcher metrics: <a href=https://github.com/cortexproject/cortex/pull/2375>#2375</a><ul><li><code>cortex_querier_bucket_store_blocks_meta_syncs_total</code> > <code>cortex_querier_blocks_meta_syncs_total</code></li><li><code>cortex_querier_bucket_store_blocks_meta_sync_failures_total</code> > <code>cortex_querier_blocks_meta_sync_failures_total</code></li><li><code>cortex_querier_bucket_store_blocks_meta_sync_duration_seconds</code> > <code>cortex_querier_blocks_meta_sync_duration_seconds</code></li><li><code>cortex_querier_bucket_store_blocks_meta_sync_consistency_delay_seconds</code> > <code>cortex_querier_blocks_meta_sync_consistency_delay_seconds</code></li></ul></li><li>[CHANGE] Experimental TSDB: Modified default values for <code>compactor.deletion-delay</code> option from 48h to 12h and <code>-experimental.tsdb.bucket-store.ignore-deletion-marks-delay</code> from 24h to 6h. <a href=https://github.com/cortexproject/cortex/pull/2414>#2414</a></li><li>[CHANGE] WAL: Default value of <code>-ingester.checkpoint-enabled</code> changed to <code>true</code>. <a href=https://github.com/cortexproject/cortex/pull/2416>#2416</a></li><li>[CHANGE] <code>trace_id</code> field in log files has been renamed to <code>traceID</code>. <a href=https://github.com/cortexproject/cortex/pull/2518>#2518</a></li><li>[CHANGE] Slow query log has a different output now. Previously used <code>url</code> field has been replaced with <code>host</code> and <code>path</code>, and query parameters are logged as individual log fields with <code>qs_</code> prefix. <a href=https://github.com/cortexproject/cortex/pull/2520>#2520</a></li><li>[CHANGE] WAL: WAL and checkpoint compression is now disabled. <a href=https://github.com/cortexproject/cortex/pull/2436>#2436</a></li><li>[CHANGE] Update in dependency <code>go-kit/kit</code> from <code>v0.9.0</code> to <code>v0.10.0</code>. HTML escaping disabled in JSON Logger. <a href=https://github.com/cortexproject/cortex/pull/2535>#2535</a></li><li>[CHANGE] Experimental TSDB: Removed <code>cortex_&lt;service>_</code> prefix from Thanos objstore metrics and added <code>component</code> label to distinguish which Cortex component is doing API calls to the object storage when running in single-binary mode: <a href=https://github.com/cortexproject/cortex/pull/2568>#2568</a><ul><li><code>cortex_&lt;service>_thanos_objstore_bucket_operations_total</code> renamed to <code>thanos_objstore_bucket_operations_total{component="&lt;name>"}</code></li><li><code>cortex_&lt;service>_thanos_objstore_bucket_operation_failures_total</code> renamed to <code>thanos_objstore_bucket_operation_failures_total{component="&lt;name>"}</code></li><li><code>cortex_&lt;service>_thanos_objstore_bucket_operation_duration_seconds</code> renamed to <code>thanos_objstore_bucket_operation_duration_seconds{component="&lt;name>"}</code></li><li><code>cortex_&lt;service>_thanos_objstore_bucket_last_successful_upload_time</code> renamed to <code>thanos_objstore_bucket_last_successful_upload_time{component="&lt;name>"}</code></li></ul></li><li>[CHANGE] FIFO cache: The <code>-&lt;prefix>.fifocache.size</code> CLI flag has been renamed to <code>-&lt;prefix>.fifocache.max-size-items</code> as well as its YAML config option <code>size</code> renamed to <code>max_size_items</code>. <a href=https://github.com/cortexproject/cortex/pull/2319>#2319</a></li><li>[FEATURE] Ruler: The <code>-ruler.evaluation-delay</code> flag was added to allow users to configure a default evaluation delay for all rules in cortex. The default value is 0 which is the current behavior. <a href=https://github.com/cortexproject/cortex/pull/2423>#2423</a></li><li>[FEATURE] Experimental: Added a new object storage client for OpenStack Swift. <a href=https://github.com/cortexproject/cortex/pull/2440>#2440</a></li><li>[FEATURE] TLS config options added to the Server. <a href=https://github.com/cortexproject/cortex/pull/2535>#2535</a></li><li>[FEATURE] Experimental: Added support for <code>/api/v1/metadata</code> Prometheus-based endpoint. <a href=https://github.com/cortexproject/cortex/pull/2549>#2549</a></li><li>[FEATURE] Add ability to limit concurrent queries to Cassandra with <code>-cassandra.query-concurrency</code> flag. <a href=https://github.com/cortexproject/cortex/pull/2562>#2562</a></li><li>[FEATURE] Experimental TSDB: Introduced store-gateway service used by the experimental blocks storage to load and query blocks. The store-gateway optionally supports blocks sharding and replication via a dedicated hash ring, configurable via <code>-experimental.store-gateway.sharding-enabled</code> and <code>-experimental.store-gateway.sharding-ring.*</code> flags. The following metrics have been added: <a href=https://github.com/cortexproject/cortex/pull/2433>#2433</a> <a href=https://github.com/cortexproject/cortex/pull/2458>#2458</a> <a href=https://github.com/cortexproject/cortex/pull/2469>#2469</a> <a href=https://github.com/cortexproject/cortex/pull/2523>#2523</a><ul><li><code>cortex_querier_storegateway_instances_hit_per_query</code></li></ul></li><li>[ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing <code>cortex_discarded_samples_total</code> metric. <a href=https://github.com/cortexproject/cortex/pull/2370>#2370</a></li><li>[ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. <a href=https://github.com/cortexproject/cortex/pull/2383>#2383</a></li><li>[ENHANCEMENT] Experimental TSDB: Added <code>cortex_querier_blocks_meta_synced</code>, which reflects current state of synced blocks over all tenants. <a href=https://github.com/cortexproject/cortex/pull/2392>#2392</a></li><li>[ENHANCEMENT] Added <code>cortex_distributor_latest_seen_sample_timestamp_seconds</code> metric to see how far behind Prometheus servers are in sending data. <a href=https://github.com/cortexproject/cortex/pull/2371>#2371</a></li><li>[ENHANCEMENT] FIFO cache to support eviction based on memory usage. Added <code>-&lt;prefix>.fifocache.max-size-bytes</code> CLI flag and YAML config option <code>max_size_bytes</code> to specify memory limit of the cache. <a href=https://github.com/cortexproject/cortex/pull/2319>#2319</a>, <a href=https://github.com/cortexproject/cortex/pull/2527>#2527</a></li><li>[ENHANCEMENT] Added <code>-querier.worker-match-max-concurrent</code>. Force worker concurrency to match the <code>-querier.max-concurrent</code> option. Overrides <code>-querier.worker-parallelism</code>. <a href=https://github.com/cortexproject/cortex/pull/2456>#2456</a></li><li>[ENHANCEMENT] Added the following metrics for monitoring delete requests: <a href=https://github.com/cortexproject/cortex/pull/2445>#2445</a><ul><li><code>cortex_purger_delete_requests_received_total</code>: Number of delete requests received per user.</li><li><code>cortex_purger_delete_requests_processed_total</code>: Number of delete requests processed per user.</li><li><code>cortex_purger_delete_requests_chunks_selected_total</code>: Number of chunks selected while building delete plans per user.</li><li><code>cortex_purger_delete_requests_processing_failures_total</code>: Number of delete requests processing failures per user.</li></ul></li><li>[ENHANCEMENT] Single Binary: Added query-frontend to the single binary. Single binary users will now benefit from various query-frontend features. Primarily: sharding, parallelization, load shedding, additional caching (if configured), and query retries. <a href=https://github.com/cortexproject/cortex/pull/2437>#2437</a></li><li>[ENHANCEMENT] Allow 1w (where w denotes week) and 1y (where y denotes year) when setting <code>-store.cache-lookups-older-than</code> and <code>-store.max-look-back-period</code>. <a href=https://github.com/cortexproject/cortex/pull/2454>#2454</a></li><li>[ENHANCEMENT] Optimize index queries for matchers using &ldquo;a|b|c&rdquo;-type regex. <a href=https://github.com/cortexproject/cortex/pull/2446>#2446</a> <a href=https://github.com/cortexproject/cortex/pull/2475>#2475</a></li><li>[ENHANCEMENT] Added per tenant metrics for queries and chunks and bytes read from chunk store: <a href=https://github.com/cortexproject/cortex/pull/2463>#2463</a><ul><li><code>cortex_chunk_store_fetched_chunks_total</code> and <code>cortex_chunk_store_fetched_chunk_bytes_total</code></li><li><code>cortex_query_frontend_queries_total</code> (per tenant queries counted by the frontend)</li></ul></li><li>[ENHANCEMENT] WAL: New metrics <code>cortex_ingester_wal_logged_bytes_total</code> and <code>cortex_ingester_checkpoint_logged_bytes_total</code> added to track total bytes logged to disk for WAL and checkpoints. <a href=https://github.com/cortexproject/cortex/pull/2497>#2497</a></li><li>[ENHANCEMENT] Add de-duplicated chunks counter <code>cortex_chunk_store_deduped_chunks_total</code> which counts every chunk not sent to the store because it was already sent by another replica. <a href=https://github.com/cortexproject/cortex/pull/2485>#2485</a></li><li>[ENHANCEMENT] Query-frontend now also logs the POST data of long queries. <a href=https://github.com/cortexproject/cortex/pull/2481>#2481</a></li><li>[ENHANCEMENT] WAL: Ingester WAL records now have type header and the custom WAL records have been replaced by Prometheus TSDB&rsquo;s WAL records. Old records will not be supported from 1.3 onwards. Note: once this is deployed, you cannot downgrade without data loss. <a href=https://github.com/cortexproject/cortex/pull/2436>#2436</a></li><li>[ENHANCEMENT] Redis Cache: Added <code>idle_timeout</code>, <code>wait_on_pool_exhaustion</code> and <code>max_conn_lifetime</code> options to redis cache configuration. <a href=https://github.com/cortexproject/cortex/pull/2550>#2550</a></li><li>[ENHANCEMENT] WAL: the experimental tag has been removed on the WAL in ingesters. <a href=https://github.com/cortexproject/cortex/pull/2560>#2560</a></li><li>[ENHANCEMENT] Use newer AWS API for paginated queries - removes &lsquo;Deprecated&rsquo; message from logfiles. <a href=https://github.com/cortexproject/cortex/pull/2452>#2452</a></li><li>[ENHANCEMENT] Experimental memberlist: Add retry with backoff on memberlist join other members. <a href=https://github.com/cortexproject/cortex/pull/2705>#2705</a></li><li>[ENHANCEMENT] Experimental TSDB: when the store-gateway sharding is enabled, unhealthy store-gateway instances are automatically removed from the ring after 10 consecutive <code>-experimental.store-gateway.sharding-ring.heartbeat-timeout</code> periods. <a href=https://github.com/cortexproject/cortex/pull/2526>#2526</a></li><li>[BUGFIX] Ruler: Ensure temporary rule files with special characters are properly mapped and cleaned up. <a href=https://github.com/cortexproject/cortex/pull/2506>#2506</a></li><li>[BUGFIX] Ensure requests are properly routed to the prometheus api embedded in the query if <code>-server.path-prefix</code> is set. Fixes <a href=https://github.com/cortexproject/cortex/pull/2411>#2411</a>. <a href=https://github.com/cortexproject/cortex/pull/2372>#2372</a></li><li>[BUGFIX] Experimental TSDB: Fixed chunk data corruption when querying back series using the experimental blocks storage. <a href=https://github.com/cortexproject/cortex/pull/2400>#2400</a></li><li>[BUGFIX] Cassandra Storage: Fix endpoint TLS host verification. <a href=https://github.com/cortexproject/cortex/pull/2109>#2109</a></li><li>[BUGFIX] Experimental TSDB: Fixed response status code from <code>422</code> to <code>500</code> when an error occurs while iterating chunks with the experimental blocks storage. <a href=https://github.com/cortexproject/cortex/pull/2402>#2402</a></li><li>[BUGFIX] Ring: Fixed a situation where upgrading from pre-1.0 cortex with a rolling strategy caused new 1.0 ingesters to lose their zone value in the ring until manually forced to re-register. <a href=https://github.com/cortexproject/cortex/pull/2404>#2404</a></li><li>[BUGFIX] Distributor: <code>/all_user_stats</code> now show API and Rule Ingest Rate correctly. <a href=https://github.com/cortexproject/cortex/pull/2457>#2457</a></li><li>[BUGFIX] Fixed <code>version</code>, <code>revision</code> and <code>branch</code> labels exported by the <code>cortex_build_info</code> metric. <a href=https://github.com/cortexproject/cortex/pull/2468>#2468</a></li><li>[BUGFIX] QueryFrontend: fixed a situation where span context missed when downstream_url is used. <a href=https://github.com/cortexproject/cortex/pull/2539>#2539</a></li><li>[BUGFIX] Querier: Fixed a situation where querier would crash because of an unresponsive frontend instance. <a href=https://github.com/cortexproject/cortex/pull/2569>#2569</a></li></ul><h2 id=101--2020-04-23>1.0.1 / 2020-04-23</h2><ul><li>[BUGFIX] Fix gaps when querying ingesters with replication factor = 3 and 2 ingesters in the cluster. <a href=https://github.com/cortexproject/cortex/pull/2503>#2503</a></li></ul><h2 id=100--2020-04-02>1.0.0 / 2020-04-02</h2><p>This is the first major release of Cortex. We made a lot of <strong>breaking changes</strong> in this release which have been detailed below. Please also see the stability guarantees we provide as part of a major release: <a href=https://cortexmetrics.io/docs/configuration/v1guarantees/>https://cortexmetrics.io/docs/configuration/v1guarantees/</a></p><ul><li><p>[CHANGE] Remove the following deprecated flags: <a href=https://github.com/cortexproject/cortex/pull/2339>#2339</a></p><ul><li><code>-metrics.error-rate-query</code> (use <code>-metrics.write-throttle-query</code> instead).</li><li><code>-store.cardinality-cache-size</code> (use <code>-store.index-cache-read.enable-fifocache</code> and <code>-store.index-cache-read.fifocache.size</code> instead).</li><li><code>-store.cardinality-cache-validity</code> (use <code>-store.index-cache-read.enable-fifocache</code> and <code>-store.index-cache-read.fifocache.duration</code> instead).</li><li><code>-distributor.limiter-reload-period</code> (flag unused)</li><li><code>-ingester.claim-on-rollout</code> (flag unused)</li><li><code>-ingester.normalise-tokens</code> (flag unused)</li></ul></li><li><p>[CHANGE] Renamed YAML file options to be more consistent. See <a href=#config-file-breaking-changes>full config file changes below</a>. <a href=https://github.com/cortexproject/cortex/pull/2273>#2273</a></p></li><li><p>[CHANGE] AWS based autoscaling has been removed. You can only use metrics based autoscaling now. <code>-applicationautoscaling.url</code> has been removed. See <a href=https://cortexmetrics.io/docs/production/aws/#dynamodb-capacity-provisioning>https://cortexmetrics.io/docs/production/aws/#dynamodb-capacity-provisioning</a> on how to migrate. <a href=https://github.com/cortexproject/cortex/pull/2328>#2328</a></p></li><li><p>[CHANGE] Renamed the <code>memcache.write-back-goroutines</code> and <code>memcache.write-back-buffer</code> flags to <code>background.write-back-concurrency</code> and <code>background.write-back-buffer</code>. This affects the following flags: <a href=https://github.com/cortexproject/cortex/pull/2241>#2241</a></p><ul><li><code>-frontend.memcache.write-back-buffer</code> &ndash;> <code>-frontend.background.write-back-buffer</code></li><li><code>-frontend.memcache.write-back-goroutines</code> &ndash;> <code>-frontend.background.write-back-concurrency</code></li><li><code>-store.index-cache-read.memcache.write-back-buffer</code> &ndash;> <code>-store.index-cache-read.background.write-back-buffer</code></li><li><code>-store.index-cache-read.memcache.write-back-goroutines</code> &ndash;> <code>-store.index-cache-read.background.write-back-concurrency</code></li><li><code>-store.index-cache-write.memcache.write-back-buffer</code> &ndash;> <code>-store.index-cache-write.background.write-back-buffer</code></li><li><code>-store.index-cache-write.memcache.write-back-goroutines</code> &ndash;> <code>-store.index-cache-write.background.write-back-concurrency</code></li><li><code>-memcache.write-back-buffer</code> &ndash;> <code>-store.chunks-cache.background.write-back-buffer</code>. Note the next change log for the difference.</li><li><code>-memcache.write-back-goroutines</code> &ndash;> <code>-store.chunks-cache.background.write-back-concurrency</code>. Note the next change log for the difference.</li></ul></li><li><p>[CHANGE] Renamed the chunk cache flags to have <code>store.chunks-cache.</code> as prefix. This means the following flags have been changed: <a href=https://github.com/cortexproject/cortex/pull/2241>#2241</a></p><ul><li><code>-cache.enable-fifocache</code> &ndash;> <code>-store.chunks-cache.cache.enable-fifocache</code></li><li><code>-default-validity</code> &ndash;> <code>-store.chunks-cache.default-validity</code></li><li><code>-fifocache.duration</code> &ndash;> <code>-store.chunks-cache.fifocache.duration</code></li><li><code>-fifocache.size</code> &ndash;> <code>-store.chunks-cache.fifocache.size</code></li><li><code>-memcache.write-back-buffer</code> &ndash;> <code>-store.chunks-cache.background.write-back-buffer</code>. Note the previous change log for the difference.</li><li><code>-memcache.write-back-goroutines</code> &ndash;> <code>-store.chunks-cache.background.write-back-concurrency</code>. Note the previous change log for the difference.</li><li><code>-memcached.batchsize</code> &ndash;> <code>-store.chunks-cache.memcached.batchsize</code></li><li><code>-memcached.consistent-hash</code> &ndash;> <code>-store.chunks-cache.memcached.consistent-hash</code></li><li><code>-memcached.expiration</code> &ndash;> <code>-store.chunks-cache.memcached.expiration</code></li><li><code>-memcached.hostname</code> &ndash;> <code>-store.chunks-cache.memcached.hostname</code></li><li><code>-memcached.max-idle-conns</code> &ndash;> <code>-store.chunks-cache.memcached.max-idle-conns</code></li><li><code>-memcached.parallelism</code> &ndash;> <code>-store.chunks-cache.memcached.parallelism</code></li><li><code>-memcached.service</code> &ndash;> <code>-store.chunks-cache.memcached.service</code></li><li><code>-memcached.timeout</code> &ndash;> <code>-store.chunks-cache.memcached.timeout</code></li><li><code>-memcached.update-interval</code> &ndash;> <code>-store.chunks-cache.memcached.update-interval</code></li><li><code>-redis.enable-tls</code> &ndash;> <code>-store.chunks-cache.redis.enable-tls</code></li><li><code>-redis.endpoint</code> &ndash;> <code>-store.chunks-cache.redis.endpoint</code></li><li><code>-redis.expiration</code> &ndash;> <code>-store.chunks-cache.redis.expiration</code></li><li><code>-redis.max-active-conns</code> &ndash;> <code>-store.chunks-cache.redis.max-active-conns</code></li><li><code>-redis.max-idle-conns</code> &ndash;> <code>-store.chunks-cache.redis.max-idle-conns</code></li><li><code>-redis.password</code> &ndash;> <code>-store.chunks-cache.redis.password</code></li><li><code>-redis.timeout</code> &ndash;> <code>-store.chunks-cache.redis.timeout</code></li></ul></li><li><p>[CHANGE] Rename the <code>-store.chunk-cache-stubs</code> to <code>-store.chunks-cache.cache-stubs</code> to be more inline with above. <a href=https://github.com/cortexproject/cortex/pull/2241>#2241</a></p></li><li><p>[CHANGE] Change prefix of flags <code>-dynamodb.periodic-table.*</code> to <code>-table-manager.index-table.*</code>. <a href=https://github.com/cortexproject/cortex/pull/2359>#2359</a></p></li><li><p>[CHANGE] Change prefix of flags <code>-dynamodb.chunk-table.*</code> to <code>-table-manager.chunk-table.*</code>. <a href=https://github.com/cortexproject/cortex/pull/2359>#2359</a></p></li><li><p>[CHANGE] Change the following flags: <a href=https://github.com/cortexproject/cortex/pull/2359>#2359</a></p><ul><li><code>-dynamodb.poll-interval</code> &ndash;> <code>-table-manager.poll-interval</code></li><li><code>-dynamodb.periodic-table.grace-period</code> &ndash;> <code>-table-manager.periodic-table.grace-period</code></li></ul></li><li><p>[CHANGE] Renamed the following flags: <a href=https://github.com/cortexproject/cortex/pull/2273>#2273</a></p><ul><li><code>-dynamodb.chunk.gang.size</code> &ndash;> <code>-dynamodb.chunk-gang-size</code></li><li><code>-dynamodb.chunk.get.max.parallelism</code> &ndash;> <code>-dynamodb.chunk-get-max-parallelism</code></li></ul></li><li><p>[CHANGE] Don&rsquo;t support mixed time units anymore for duration. For example, 168h5m0s doesn&rsquo;t work anymore, please use just one unit (s|m|h|d|w|y). <a href=https://github.com/cortexproject/cortex/pull/2252>#2252</a></p></li><li><p>[CHANGE] Utilize separate protos for rule state and storage. Experimental ruler API will not be functional until the rollout is complete. <a href=https://github.com/cortexproject/cortex/pull/2226>#2226</a></p></li><li><p>[CHANGE] Frontend worker in querier now starts after all Querier module dependencies are started. This fixes issue where frontend worker started to send queries to querier before it was ready to serve them (mostly visible when using experimental blocks storage). <a href=https://github.com/cortexproject/cortex/pull/2246>#2246</a></p></li><li><p>[CHANGE] Lifecycler component now enters Failed state on errors, and doesn&rsquo;t exit the process. (Important if you&rsquo;re vendoring Cortex and use Lifecycler) <a href=https://github.com/cortexproject/cortex/pull/2251>#2251</a></p></li><li><p>[CHANGE] <code>/ready</code> handler now returns 200 instead of 204. <a href=https://github.com/cortexproject/cortex/pull/2330>#2330</a></p></li><li><p>[CHANGE] Better defaults for the following options: <a href=https://github.com/cortexproject/cortex/pull/2344>#2344</a></p><ul><li><code>-&lt;prefix>.consul.consistent-reads</code>: Old default: <code>true</code>, new default: <code>false</code>. This reduces the load on Consul.</li><li><code>-&lt;prefix>.consul.watch-rate-limit</code>: Old default: 0, new default: 1. This rate limits the reads to 1 per second. Which is good enough for ring watches.</li><li><code>-distributor.health-check-ingesters</code>: Old default: <code>false</code>, new default: <code>true</code>.</li><li><code>-ingester.max-stale-chunk-idle</code>: Old default: 0, new default: 2m. This lets us expire series that we know are stale early.</li><li><code>-ingester.spread-flushes</code>: Old default: false, new default: true. This allows to better de-duplicate data and use less space.</li><li><code>-ingester.chunk-age-jitter</code>: Old default: 20mins, new default: 0. This is to enable the <code>-ingester.spread-flushes</code> to true.</li><li><code>-&lt;prefix>.memcached.batchsize</code>: Old default: 0, new default: 1024. This allows batching of requests and keeps the concurrent requests low.</li><li><code>-&lt;prefix>.memcached.consistent-hash</code>: Old default: false, new default: true. This allows for better cache hits when the memcaches are scaled up and down.</li><li><code>-querier.batch-iterators</code>: Old default: false, new default: true.</li><li><code>-querier.ingester-streaming</code>: Old default: false, new default: true.</li></ul></li><li><p>[CHANGE] Experimental TSDB: Added <code>-experimental.tsdb.bucket-store.postings-cache-compression-enabled</code> to enable postings compression when storing to cache. <a href=https://github.com/cortexproject/cortex/pull/2335>#2335</a></p></li><li><p>[CHANGE] Experimental TSDB: Added <code>-compactor.deletion-delay</code>, which is time before a block marked for deletion is deleted from bucket. If not 0, blocks will be marked for deletion and compactor component will delete blocks marked for deletion from the bucket. If delete-delay is 0, blocks will be deleted straight away. Note that deleting blocks immediately can cause query failures, if store gateway / querier still has the block loaded, or compactor is ignoring the deletion because it&rsquo;s compacting the block at the same time. Default value is 48h. <a href=https://github.com/cortexproject/cortex/pull/2335>#2335</a></p></li><li><p>[CHANGE] Experimental TSDB: Added <code>-experimental.tsdb.bucket-store.index-cache.postings-compression-enabled</code>, to set duration after which the blocks marked for deletion will be filtered out while fetching blocks used for querying. This option allows querier to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet. Default is 24h, half of the default value for <code>-compactor.deletion-delay</code>. <a href=https://github.com/cortexproject/cortex/pull/2335>#2335</a></p></li><li><p>[CHANGE] Experimental TSDB: Added <code>-experimental.tsdb.bucket-store.index-cache.memcached.max-item-size</code> to control maximum size of item that is stored to memcached. Defaults to 1 MiB. <a href=https://github.com/cortexproject/cortex/pull/2335>#2335</a></p></li><li><p>[FEATURE] Added experimental storage API to the ruler service that is enabled when the <code>-experimental.ruler.enable-api</code> is set to true <a href=https://github.com/cortexproject/cortex/pull/2269>#2269</a></p><ul><li><code>-ruler.storage.type</code> flag now allows <code>s3</code>,<code>gcs</code>, and <code>azure</code> values</li><li><code>-ruler.storage.(s3|gcs|azure)</code> flags exist to allow the configuration of object clients set for rule storage</li></ul></li><li><p>[CHANGE] Renamed table manager metrics. <a href=https://github.com/cortexproject/cortex/pull/2307>#2307</a> <a href=https://github.com/cortexproject/cortex/pull/2359>#2359</a></p><ul><li><code>cortex_dynamo_sync_tables_seconds</code> -> <code>cortex_table_manager_sync_duration_seconds</code></li><li><code>cortex_dynamo_table_capacity_units</code> -> <code>cortex_table_capacity_units</code></li></ul></li><li><p>[FEATURE] Flusher target to flush the WAL. <a href=https://github.com/cortexproject/cortex/pull/2075>#2075</a></p><ul><li><code>-flusher.wal-dir</code> for the WAL directory to recover from.</li><li><code>-flusher.concurrent-flushes</code> for number of concurrent flushes.</li><li><code>-flusher.flush-op-timeout</code> is duration after which a flush should timeout.</li></ul></li><li><p>[FEATURE] Ingesters can now have an optional availability zone set, to ensure metric replication is distributed across zones. This is set via the <code>-ingester.availability-zone</code> flag or the <code>availability_zone</code> field in the config file. <a href=https://github.com/cortexproject/cortex/pull/2317>#2317</a></p></li><li><p>[ENHANCEMENT] Better reuse of connections to DynamoDB and S3. <a href=https://github.com/cortexproject/cortex/pull/2268>#2268</a></p></li><li><p>[ENHANCEMENT] Reduce number of goroutines used while executing a single index query. <a href=https://github.com/cortexproject/cortex/pull/2280>#2280</a></p></li><li><p>[ENHANCEMENT] Experimental TSDB: Add support for local <code>filesystem</code> backend. <a href=https://github.com/cortexproject/cortex/pull/2245>#2245</a></p></li><li><p>[ENHANCEMENT] Experimental TSDB: Added memcached support for the TSDB index cache. <a href=https://github.com/cortexproject/cortex/pull/2290>#2290</a></p></li><li><p>[ENHANCEMENT] Experimental TSDB: Removed gRPC server to communicate between querier and BucketStore. <a href=https://github.com/cortexproject/cortex/pull/2324>#2324</a></p></li><li><p>[ENHANCEMENT] Allow 1w (where w denotes week) and 1y (where y denotes year) when setting table period and retention. <a href=https://github.com/cortexproject/cortex/pull/2252>#2252</a></p></li><li><p>[ENHANCEMENT] Added FIFO cache metrics for current number of entries and memory usage. <a href=https://github.com/cortexproject/cortex/pull/2270>#2270</a></p></li><li><p>[ENHANCEMENT] Output all config fields to /config API, including those with empty value. <a href=https://github.com/cortexproject/cortex/pull/2209>#2209</a></p></li><li><p>[ENHANCEMENT] Add &ldquo;missing_metric_name&rdquo; and &ldquo;metric_name_invalid&rdquo; reasons to cortex_discarded_samples_total metric. <a href=https://github.com/cortexproject/cortex/pull/2346>#2346</a></p></li><li><p>[ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing <code>cortex_discarded_samples_total</code> metric. <a href=https://github.com/cortexproject/cortex/pull/2370>#2370</a></p></li><li><p>[BUGFIX] Ensure user state metrics are updated if a transfer fails. <a href=https://github.com/cortexproject/cortex/pull/2338>#2338</a></p></li><li><p>[BUGFIX] Fixed etcd client keepalive settings. <a href=https://github.com/cortexproject/cortex/pull/2278>#2278</a></p></li><li><p>[BUGFIX] Register the metrics of the WAL. <a href=https://github.com/cortexproject/cortex/pull/2295>#2295</a></p></li><li><p>[BUXFIX] Experimental TSDB: fixed error handling when ingesting out of bound samples. <a href=https://github.com/cortexproject/cortex/pull/2342>#2342</a></p></li></ul><h3 id=known-issues-1>Known issues</h3><ul><li>This experimental blocks storage in Cortex <code>1.0.0</code> has a bug which may lead to the error <code>cannot iterate chunk for series</code> when running queries. This bug has been fixed in <a href=https://github.com/cortexproject/cortex/pull/2400>#2400</a>. If you&rsquo;re running the experimental blocks storage, please build Cortex from <code>master</code>.</li></ul><h3 id=config-file-breaking-changes>Config file breaking changes</h3><p>In this section you can find a config file diff showing the breaking changes introduced in Cortex. You can also find the <a href=https://cortexmetrics.io/docs/configuration/configuration-file/>full configuration file reference doc</a> in the website.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-diff data-lang=diff><span style=display:flex><span>### ingester_config
 </span></span><span style=display:flex><span>
 </span></span><span style=display:flex><span> # Period with which to attempt to flush chunks.
diff --git a/docs/code-of-conduct/index.html b/docs/code-of-conduct/index.html
index c337fa10bb..3a035a998e 100644
--- a/docs/code-of-conduct/index.html
+++ b/docs/code-of-conduct/index.html
@@ -2,7 +2,7 @@
 "><meta property="og:title" content="Code of Conduct"><meta property="og:description" content="Cortex follows the CNCF Code of Conduct."><meta property="og:type" content="article"><meta property="og:url" content="/docs/code-of-conduct/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Code of Conduct"><meta itemprop=description content="Cortex follows the CNCF Code of Conduct."><meta itemprop=wordCount content="7"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Code of Conduct"><meta name=twitter:description content="Cortex follows the CNCF Code of Conduct."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span class=td-sidebar-nav-active-item>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/code-of-conduct.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Code%20of%20Conduct" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/code-of-conduct/ aria-disabled=true class="btn-link disabled">Code of Conduct</a></li></ol></nav><div class=td-content><h1>Code of Conduct</h1><header class=article-meta></header><p>Cortex follows the <a href=https://github.com/cncf/foundation/blob/master/code-of-conduct.md>CNCF Code of Conduct</a>.</p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/configuration/arguments/index.html b/docs/configuration/arguments/index.html
index 55bf494560..ea35f8138b 100644
--- a/docs/configuration/arguments/index.html
+++ b/docs/configuration/arguments/index.html
@@ -5,7 +5,7 @@
 Duration arguments should be specified with a unit like 5s or 3h."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span class=td-sidebar-nav-active-item>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/configuration/arguments.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Cortex%20Arguments" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#general-notes>General Notes</a></li><li><a href=#querier>Querier</a></li><li><a href=#querier-and-ruler>Querier and Ruler</a></li><li><a href=#query-frontend>Query Frontend</a></li><li><a href=#distributor>Distributor</a><ul><li><a href=#ringha-tracker-store>Ring/HA Tracker Store</a></li><li><a href=#ha-tracker>HA Tracker</a></li></ul></li><li><a href=#ingester>Ingester</a></li><li><a href=#runtime-configuration-file>Runtime Configuration file</a></li><li><a href=#ingester-distributor--querier-limits>Ingester, Distributor & Querier limits.</a></li><li><a href=#ingester-instance-limits>Ingester Instance Limits</a></li><li><a href=#storage>Storage</a></li><li><a href=#dns-service-discovery>DNS Service Discovery</a><ul><li><a href=#supported-discovery-modes>Supported discovery modes</a></li></ul></li><li><a href=#logging-of-ip-of-reverse-proxy>Logging of IP of reverse proxy</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/configuration/>Configuration</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/configuration/arguments/ aria-disabled=true class="btn-link disabled">Cortex Arguments Explained</a></li></ol></nav><div class=td-content><h1>Cortex Arguments</h1><header class=article-meta></header><h2 id=general-notes>General Notes</h2><p>Cortex has evolved over several years, and the command-line options sometimes reflect this heritage. In some cases the default value for options is not the recommended value, and in some cases names do not reflect the true meaning. We do intend to clean this up, but it requires a lot of care to avoid breaking existing installations. In the meantime we regret the inconvenience.</p><p>Duration arguments should be specified with a unit like <code>5s</code> or <code>3h</code>. Valid time units are &ldquo;ms&rdquo;, &ldquo;s&rdquo;, &ldquo;m&rdquo;, &ldquo;h&rdquo;.</p><h2 id=querier>Querier</h2><ul><li><p><code>-querier.max-concurrent</code></p><p>The maximum number of top-level PromQL queries that will execute at the same time, per querier process.
diff --git a/docs/configuration/configuration-file/index.html b/docs/configuration/configuration-file/index.html
index 55269dfe52..d199d836e1 100644
--- a/docs/configuration/configuration-file/index.html
+++ b/docs/configuration/configuration-file/index.html
@@ -3,12 +3,12 @@
 The current configuration of any Cortex component can be seen by visiting the /config HTTP path. Passwords are filtered out of this endpoint.
 Reference To specify which configuration file to load, pass the -config.file flag at the command line."><meta property="og:type" content="article"><meta property="og:url" content="/docs/configuration/configuration-file/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Configuration file"><meta itemprop=description content="Cortex can be configured using a YAML file - specified using the -config.file flag - or CLI flags. In case you combine both, CLI flags take precedence over the YAML config file.
 The current configuration of any Cortex component can be seen by visiting the /config HTTP path. Passwords are filtered out of this endpoint.
-Reference To specify which configuration file to load, pass the -config.file flag at the command line."><meta itemprop=wordCount content="31196"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Configuration file"><meta name=twitter:description content="Cortex can be configured using a YAML file - specified using the -config.file flag - or CLI flags. In case you combine both, CLI flags take precedence over the YAML config file.
+Reference To specify which configuration file to load, pass the -config.file flag at the command line."><meta itemprop=wordCount content="31328"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Configuration file"><meta name=twitter:description content="Cortex can be configured using a YAML file - specified using the -config.file flag - or CLI flags. In case you combine both, CLI flags take precedence over the YAML config file.
 The current configuration of any Cortex component can be seen by visiting the /config HTTP path. Passwords are filtered out of this endpoint.
 Reference To specify which configuration file to load, pass the -config.file flag at the command line."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span class=td-sidebar-nav-active-item>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/configuration/config-file-reference.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Configuration%20file" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#reference>Reference</a><ul><li><a href=#generic-placeholders>Generic placeholders</a></li><li><a href=#use-environment-variables-in-the-configuration>Use environment variables in the configuration</a></li><li><a href=#supported-contents-and-default-values-of-the-config-file>Supported contents and default values of the config file</a></li><li><a href=#alertmanager_config><code>alertmanager_config</code></a></li><li><a href=#alertmanager_storage_config><code>alertmanager_storage_config</code></a></li><li><a href=#blocks_storage_config><code>blocks_storage_config</code></a></li><li><a href=#compactor_config><code>compactor_config</code></a></li><li><a href=#configs_config><code>configs_config</code></a></li><li><a href=#configstore_config><code>configstore_config</code></a></li><li><a href=#consul_config><code>consul_config</code></a></li><li><a href=#distributor_config><code>distributor_config</code></a></li><li><a href=#etcd_config><code>etcd_config</code></a></li><li><a href=#fifo_cache_config><code>fifo_cache_config</code></a></li><li><a href=#flusher_config><code>flusher_config</code></a></li><li><a href=#frontend_worker_config><code>frontend_worker_config</code></a></li><li><a href=#ingester_config><code>ingester_config</code></a></li><li><a href=#ingester_client_config><code>ingester_client_config</code></a></li><li><a href=#limits_config><code>limits_config</code></a></li><li><a href=#memberlist_config><code>memberlist_config</code></a></li><li><a href=#memcached_config><code>memcached_config</code></a></li><li><a href=#memcached_client_config><code>memcached_client_config</code></a></li><li><a href=#querier_config><code>querier_config</code></a></li><li><a href=#query_frontend_config><code>query_frontend_config</code></a></li><li><a href=#query_range_config><code>query_range_config</code></a></li><li><a href=#redis_config><code>redis_config</code></a></li><li><a href=#ruler_config><code>ruler_config</code></a></li><li><a href=#ruler_storage_config><code>ruler_storage_config</code></a></li><li><a href=#runtime_configuration_storage_config><code>runtime_configuration_storage_config</code></a></li><li><a href=#s3_sse_config><code>s3_sse_config</code></a></li><li><a href=#server_config><code>server_config</code></a></li><li><a href=#storage_config><code>storage_config</code></a></li><li><a href=#store_gateway_config><code>store_gateway_config</code></a></li><li><a href=#tracing_config><code>tracing_config</code></a></li><li><a href=#limitsperlabelset><code>LimitsPerLabelSet</code></a></li><li><a href=#prioritydef><code>PriorityDef</code></a></li><li><a href=#queryattribute><code>QueryAttribute</code></a></li><li><a href=#disabledrulegroup><code>DisabledRuleGroup</code></a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/configuration/>Configuration</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/configuration/configuration-file/ aria-disabled=true class="btn-link disabled">Configuration file</a></li></ol></nav><div class=td-content><h1>Configuration file</h1><header class=article-meta></header><p>Cortex can be configured using a YAML file - specified using the <code>-config.file</code> flag - or CLI flags. In case you combine both, CLI flags take precedence over the YAML config file.</p><p>The current configuration of any Cortex component can be seen by visiting the <code>/config</code> HTTP path.
@@ -1892,6 +1892,14 @@
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#204a87;font-weight:700>[lazy_expanded_postings_enabled</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;boolean&gt; | default = false]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># Mark posting group as lazy if it fetches more keys than R * max series the</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># query should fetch. With R set to 100, a posting group which fetches 100K</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># keys will be marked as lazy if the current query only fetches 1000 series.</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># This config is only valid if lazy expanded posting is enabled. 0 disables</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># the limit.</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># CLI flag: -blocks-storage.bucket-store.lazy-expanded-posting-group-max-key-series-ratio</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#204a87;font-weight:700>[lazy_expanded_posting_group_max_key_series_ratio</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;float&gt; | default = 100]</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># Controls how many series to fetch per batch in Store Gateway. Default value</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># is 10000.</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># CLI flag: -blocks-storage.bucket-store.series-batch-size</span><span style=color:#f8f8f8;text-decoration:underline>
@@ -2585,6 +2593,13 @@
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># CLI flag: -distributor.ring.instance-interface-names</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#204a87;font-weight:700>[instance_interface_names</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;list of string&gt; | default = [eth0 en0]]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#8f5902;font-style:italic># EXPERIMENTAL: Number of go routines to handle push calls from distributors to</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#8f5902;font-style:italic># ingesters. When no workers are available, a new goroutine will be spawned</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#8f5902;font-style:italic># automatically. If set to 0 (default), workers are disabled, and a new</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#8f5902;font-style:italic># goroutine will be created for each push request.</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#8f5902;font-style:italic># CLI flag: -distributor.num-push-workers</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#204a87;font-weight:700>[num_push_workers</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;int&gt; | default = 0]</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#204a87;font-weight:700>instance_limits</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># Max ingestion rate (samples/sec) that this distributor will accept. This</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#8f5902;font-style:italic># limit is per-distributor, not per-tenant. Additional push requests will be</span><span style=color:#f8f8f8;text-decoration:underline>
@@ -3082,17 +3097,17 @@
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#204a87;font-weight:700>healthcheck_config</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># The number of consecutive failed health checks required before considering</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># a target unhealthy. 0 means disabled.</span><span style=color:#f8f8f8;text-decoration:underline>
-</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -ingester.client.unhealthy-threshold</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -ingester.client.healthcheck.unhealthy-threshold</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#204a87;font-weight:700>[unhealthy_threshold</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;int&gt; | default = 0]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># The approximate amount of time between health checks of an individual</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># target.</span><span style=color:#f8f8f8;text-decoration:underline>
-</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -ingester.client.interval</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -ingester.client.healthcheck.interval</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#204a87;font-weight:700>[interval</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;duration&gt; | default = 5s]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># The amount of time during which no response from a target means a failed</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># health check.</span><span style=color:#f8f8f8;text-decoration:underline>
-</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -ingester.client.timeout</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -ingester.client.healthcheck.timeout</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#204a87;font-weight:700>[timeout</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;duration&gt; | default = 1s]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#8f5902;font-style:italic># Max inflight push requests that this ingester client can handle. This limit is</span><span style=color:#f8f8f8;text-decoration:underline>
@@ -3795,17 +3810,17 @@
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>  </span><span style=color:#204a87;font-weight:700>healthcheck_config</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># The number of consecutive failed health checks required before considering</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># a target unhealthy. 0 means disabled.</span><span style=color:#f8f8f8;text-decoration:underline>
-</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -querier.store-gateway-client.unhealthy-threshold</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -querier.store-gateway-client.healthcheck.unhealthy-threshold</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#204a87;font-weight:700>[unhealthy_threshold</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;int&gt; | default = 0]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># The approximate amount of time between health checks of an individual</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># target.</span><span style=color:#f8f8f8;text-decoration:underline>
-</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -querier.store-gateway-client.interval</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -querier.store-gateway-client.healthcheck.interval</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#204a87;font-weight:700>[interval</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;duration&gt; | default = 5s]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># The amount of time during which no response from a target means a failed</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># health check.</span><span style=color:#f8f8f8;text-decoration:underline>
-</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -querier.store-gateway-client.timeout</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#8f5902;font-style:italic># CLI flag: -querier.store-gateway-client.healthcheck.timeout</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>    </span><span style=color:#204a87;font-weight:700>[timeout</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>&lt;duration&gt; | default = 1s]</span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline>
 </span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#8f5902;font-style:italic># If enabled, store gateway query stats will be logged using `info` log level.</span><span style=color:#f8f8f8;text-decoration:underline>
diff --git a/docs/configuration/index.html b/docs/configuration/index.html
index 07ac31d88e..67e37faf33 100644
--- a/docs/configuration/index.html
+++ b/docs/configuration/index.html
@@ -1,7 +1,7 @@
 <!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.101.0"><link rel=alternate type=application/rss+xml href=/docs/configuration/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/favicons/android-96x96.png sizes=96x96><link rel=icon type=image/png href=/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/favicons/android-192x192.png sizes=192x192><title>Configuration | Cortex</title><meta name=description content><meta property="og:title" content="Configuration"><meta property="og:description" content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta property="og:type" content="website"><meta property="og:url" content="/docs/configuration/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Configuration"><meta itemprop=description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Configuration"><meta name=twitter:description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span class=td-sidebar-nav-active-item>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/configuration/_index.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Configuration" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/configuration/ aria-disabled=true class="btn-link disabled">Configuration</a></li></ol></nav><div class=td-content><h1>Configuration</h1><header class=article-meta></header><ul><li><a href=/docs/configuration/configuration-file/>Configuration file</a></li><li><a href=/docs/configuration/arguments/>Cortex Arguments</a></li><li><a href=/docs/configuration/prometheus-frontend/>Prometheus Frontend</a></li><li><a href=/docs/configuration/v1guarantees/>v1.x Guarantees</a></li></ul></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/configuration/index.xml b/docs/configuration/index.xml
index 752424f0ff..6087fa1c6f 100644
--- a/docs/configuration/index.xml
+++ b/docs/configuration/index.xml
@@ -1909,6 +1909,14 @@ References to undefined variables are replaced by empty strings unless you speci
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[lazy_expanded_postings_enabled&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;boolean&amp;gt; | default = false]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Mark posting group as lazy if it fetches more keys than R * max series the&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># query should fetch. With R set to 100, a posting group which fetches 100K&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># keys will be marked as lazy if the current query only fetches 1000 series.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># This config is only valid if lazy expanded posting is enabled. 0 disables&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># the limit.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.lazy-expanded-posting-group-max-key-series-ratio&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[lazy_expanded_posting_group_max_key_series_ratio&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;float&amp;gt; | default = 100]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Controls how many series to fetch per batch in Store Gateway. Default value&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># is 10000.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.series-batch-size&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
@@ -2627,6 +2635,13 @@ References to undefined variables are replaced by empty strings unless you speci
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -distributor.ring.instance-interface-names&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[instance_interface_names&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;list of string&amp;gt; | default = [eth0 en0]]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># EXPERIMENTAL: Number of go routines to handle push calls from distributors to&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># ingesters. When no workers are available, a new goroutine will be spawned&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># automatically. If set to 0 (default), workers are disabled, and a new&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># goroutine will be created for each push request.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -distributor.num-push-workers&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#204a87;font-weight:bold">[num_push_workers&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;int&amp;gt; | default = 0]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#204a87;font-weight:bold">instance_limits&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Max ingestion rate (samples/sec) that this distributor will accept. This&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># limit is per-distributor, not per-tenant. Additional push requests will be&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
@@ -3146,17 +3161,17 @@ References to undefined variables are replaced by empty strings unless you speci
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">healthcheck_config&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The number of consecutive failed health checks required before considering&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># a target unhealthy. 0 means disabled.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -ingester.client.unhealthy-threshold&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -ingester.client.healthcheck.unhealthy-threshold&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[unhealthy_threshold&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;int&amp;gt; | default = 0]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The approximate amount of time between health checks of an individual&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># target.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -ingester.client.interval&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -ingester.client.healthcheck.interval&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[interval&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;duration&amp;gt; | default = 5s]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The amount of time during which no response from a target means a failed&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># health check.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -ingester.client.timeout&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -ingester.client.healthcheck.timeout&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[timeout&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;duration&amp;gt; | default = 1s]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Max inflight push requests that this ingester client can handle. This limit is&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
@@ -3869,17 +3884,17 @@ References to undefined variables are replaced by empty strings unless you speci
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">healthcheck_config&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The number of consecutive failed health checks required before considering&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># a target unhealthy. 0 means disabled.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.unhealthy-threshold&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.healthcheck.unhealthy-threshold&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[unhealthy_threshold&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;int&amp;gt; | default = 0]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The approximate amount of time between health checks of an individual&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># target.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.interval&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.healthcheck.interval&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[interval&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;duration&amp;gt; | default = 5s]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The amount of time during which no response from a target means a failed&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># health check.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.timeout&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.healthcheck.timeout&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[timeout&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;duration&amp;gt; | default = 1s]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># If enabled, store gateway query stats will be logged using `info` log level.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
diff --git a/docs/configuration/prometheus-frontend/index.html b/docs/configuration/prometheus-frontend/index.html
index 07382ae4c8..33baa0fa6f 100644
--- a/docs/configuration/prometheus-frontend/index.html
+++ b/docs/configuration/prometheus-frontend/index.html
@@ -6,7 +6,7 @@
 # Disable the requirement that every request to Cortex has a # X-Scope-OrgID header. `fake` will be substituted in instead. auth_enabled: false # We only want to run the query-frontend module. target: query-frontend # We don't want the usual /api/prom prefix. http_prefix: server: http_listen_port: 9091 query_range: split_queries_by_interval: 24h align_queries_with_step: true cache_results: true # list of request headers forwarded by query frontend to downstream queriers."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span class=td-sidebar-nav-active-item>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/configuration/prometheus-frontend.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Prometheus%20Frontend" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/configuration/>Configuration</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/configuration/prometheus-frontend/ aria-disabled=true class="btn-link disabled">Prometheus Frontend</a></li></ol></nav><div class=td-content><h1>Prometheus Frontend</h1><header class=article-meta></header><p>You can use the Cortex query frontend with any Prometheus-API compatible
diff --git a/docs/configuration/v1guarantees/index.html b/docs/configuration/v1guarantees/index.html
index fe7938b0b6..c993cba065 100644
--- a/docs/configuration/v1guarantees/index.html
+++ b/docs/configuration/v1guarantees/index.html
@@ -6,7 +6,7 @@
 Flags, Config and minor version upgrades Upgrading cortex from one minor version to the next should &ldquo;just work&rdquo;; that being said, we don&rsquo;t want to bump the major version every time we remove a flag, so we will will keep deprecated flags around for 2 minor release. There is a metric (deprecated_flags_inuse_total) you can alert on to find out if you&rsquo;re using a deprecated flag."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span class=td-sidebar-nav-active-item>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/configuration/v1-guarantees.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=v1.x%20Guarantees" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#flags-config-and-minor-version-upgrades>Flags, Config and minor version upgrades</a></li><li><a href=#reading-old-data>Reading old data</a></li><li><a href=#api-compatibility>API Compatibility</a></li><li><a href=#experimental-features>Experimental features</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/configuration/>Configuration</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/configuration/v1guarantees/ aria-disabled=true class="btn-link disabled">v1.x Guarantees</a></li></ol></nav><div class=td-content><h1>v1.x Guarantees</h1><header class=article-meta></header><p>For the v1.0 release, we want to provide the following guarantees:</p><h2 id=flags-config-and-minor-version-upgrades>Flags, Config and minor version upgrades</h2><p>Upgrading cortex from one minor version to the next should &ldquo;just work&rdquo;; that being said, we don&rsquo;t want to bump the major version every time we remove a flag, so we will will keep deprecated flags around for 2 minor release. There is a metric (<code>deprecated_flags_inuse_total</code>) you can alert on to find out if you&rsquo;re using a deprecated flag.</p><p>Similarly to flags, minor version upgrades using config files should &ldquo;just work&rdquo;. If we do need to change config, we will keep the old way working for two minor version. There will be a metric you can alert on for this too.</p><p>These guarantees don&rsquo;t apply for <a href=#experimental-features>experimental features</a>.</p><h2 id=reading-old-data>Reading old data</h2><p>The Cortex maintainers commit to ensuring future version of Cortex can read data written by versions up to two years old. In practice we expect to be able to read more, but this is our guarantee.</p><h2 id=api-compatibility>API Compatibility</h2><p>Cortex strives to be 100% API compatible with Prometheus (under <code>/prometheus/*</code> and <code>/api/prom/*</code>); any deviation from this is considered a bug, except:</p><ul><li>Additional API endpoints for creating, removing and modifying alerts and recording rules.</li><li>Additional API around pushing metrics (under <code>/api/push</code>).</li><li>Additional API endpoints for management of Cortex itself, such as the ring. These APIs are not part of the any compatibility guarantees.</li></ul><p><em>For more information, please refer to the <a href=/docs/guides/limitations/>limitations</a> doc.</em></p><h2 id=experimental-features>Experimental features</h2><p>Cortex is an actively developed project and we want to encourage the introduction of new features and capability. As such, not everything in each release of Cortex is considered &ldquo;production-ready&rdquo;. We don&rsquo;t provide any backwards compatibility guarantees on these and the config and flags might break.</p><p>Currently experimental features are:</p><ul><li>Ruler: Evaluate rules to query frontend instead of ingesters (enabled via <code>-ruler.frontend-address</code> )</li><li>S3 Server Side Encryption (SSE) using KMS (including per-tenant KMS config overrides).</li><li>Azure blob storage.</li><li>Zone awareness based replication.</li><li>Ruler API (to PUT rules).</li><li>Alertmanager:<ul><li>API (enabled via <code>-experimental.alertmanager.enable-api</code>)</li><li>Sharding of tenants across multiple instances (enabled via <code>-alertmanager.sharding-enabled</code>)</li><li>Receiver integrations firewall (configured via <code>-alertmanager.receivers-firewall.*</code>)</li></ul></li><li>Memcached client DNS-based service discovery.</li><li>In-memory (FIFO) and Redis cache.</li><li>gRPC Store.</li><li>TLS configuration in gRPC and HTTP clients.</li><li>TLS configuration in Etcd client.</li><li>Blocksconvert tools</li><li>OpenStack Swift storage support.</li><li>Metric relabeling in the distributor.</li><li>Scalable query-frontend (when using query-scheduler)</li><li>Ingester: do not unregister from ring on shutdown (<code>-ingester.unregister-on-shutdown=false</code>)</li><li>Distributor:<ul><li>Do not extend writes on unhealthy ingesters (<code>-distributor.extend-writes=false</code>)</li><li>Accept multiple HA pairs in the same request (enabled via <code>-experimental.distributor.ha-tracker.mixed-ha-samples=true</code>)</li></ul></li><li>Tenant Deletion in Purger, for blocks storage.</li><li>Query-frontend: query stats tracking (<code>-frontend.query-stats-enabled</code>)</li><li>Blocks storage bucket index<ul><li>The bucket index support in the querier and store-gateway (enabled via <code>-blocks-storage.bucket-store.bucket-index.enabled=true</code>) is experimental</li><li>The block deletion marks migration support in the compactor (<code>-compactor.block-deletion-marks-migration-enabled</code>) is temporarily and will be removed in future versions</li></ul></li><li>Querier: tenant federation</li><li>The thanosconvert tool for converting Thanos block metadata to Cortex</li><li>HA Tracker: cleanup of old replicas from KV Store.</li><li>Instance limits in ingester and distributor</li><li>Exemplar storage, currently in-memory only within the Ingester based on Prometheus exemplar storage (<code>-blocks-storage.tsdb.max-exemplars</code>)</li><li>Querier limits:<ul><li><code>-querier.max-fetched-chunks-per-query</code></li><li><code>-querier.max-fetched-chunk-bytes-per-query</code></li><li><code>-querier.max-fetched-series-per-query</code></li></ul></li><li>Alertmanager limits<ul><li>notification rate (<code>-alertmanager.notification-rate-limit</code> and <code>-alertmanager.notification-rate-limit-per-integration</code>)</li><li>dispatcher groups (<code>-alertmanager.max-dispatcher-aggregation-groups</code>)</li><li>user config size (<code>-alertmanager.max-config-size-bytes</code>)</li><li>templates count in user config (<code>-alertmanager.max-templates-count</code>)</li><li>max template size (<code>-alertmanager.max-template-size-bytes</code>)</li></ul></li><li>Disabling ring heartbeat timeouts<ul><li><code>-distributor.ring.heartbeat-timeout=0</code></li><li><code>-ring.heartbeat-timeout=0</code></li><li><code>-ruler.ring.heartbeat-timeout=0</code></li><li><code>-alertmanager.sharding-ring.heartbeat-timeout=0</code></li><li><code>-compactor.ring.heartbeat-timeout=0</code></li><li><code>-store-gateway.sharding-ring.heartbeat-timeout=0</code></li></ul></li><li>Disabling ring heartbeats<ul><li><code>-distributor.ring.heartbeat-period=0</code></li><li><code>-ingester.heartbeat-period=0</code></li><li><code>-ruler.ring.heartbeat-period=0</code></li><li><code>-alertmanager.sharding-ring.heartbeat-period=0</code></li><li><code>-compactor.ring.heartbeat-period=0</code></li><li><code>-store-gateway.sharding-ring.heartbeat-period=0</code></li></ul></li><li>Compactor shuffle sharding<ul><li>Enabled via <code>-compactor.sharding-enabled=true</code>, <code>-compactor.sharding-strategy=shuffle-sharding</code>, and <code>-compactor.tenant-shard-size</code> set to a value larger than 0.</li></ul></li><li>Vertical sharding at query frontend for range/instant queries<ul><li><code>-frontend.query-vertical-shard-size</code> (int) CLI flag</li><li><code>query_vertical_shard_size</code> (int) field in runtime config file</li></ul></li><li>Snapshotting of in-memory TSDB on disk during shutdown<ul><li><code>-blocks-storage.tsdb.memory-snapshot-on-shutdown</code> (boolean) CLI flag</li></ul></li><li>Out of order samples support<ul><li><code>-blocks-storage.tsdb.out-of-order-cap-max</code> (int) CLI flag</li><li><code>-ingester.out-of-order-time-window</code> (duration) CLI flag</li><li><code>out_of_order_time_window</code> (duration) field in runtime config file</li></ul></li><li>Store Gateway Zone Stable Shuffle Sharding<ul><li><code>-store-gateway.sharding-ring.zone-stable-shuffle-sharding</code> CLI flag</li><li><code>zone_stable_shuffle_sharding</code> (boolean) field in config file</li></ul></li><li>Basic Lifecycler (Storegateway, Alertmanager, Ruler) Final Sleep on shutdown, which tells the pod wait before shutdown, allowing a delay to propagate ring changes.<ul><li><code>-ruler.ring.final-sleep</code> (duration) CLI flag</li><li><code>store-gateway.sharding-ring.final-sleep</code> (duration) CLI flag</li><li><code>alertmanager-sharding-ring.final-sleep</code> (duration) CLI flag</li></ul></li><li>OTLP Receiver</li><li>Persistent tokens in the Ruler Ring:<ul><li><code>-ruler.ring.tokens-file-path</code> (path) CLI flag</li></ul></li><li>Native Histograms<ul><li>Ingestion can be enabled by setting <code>-blocks-storage.tsdb.enable-native-histograms=true</code> on Ingester.</li></ul></li><li>String interning for metrics labels<ul><li>Enable string interning for metrics labels by setting <code>-ingester.labels-string-interning-enabled</code> on Ingester.</li></ul></li><li>Query-frontend: query rejection (<code>-frontend.query-rejection.enabled</code>)</li><li>Querier: protobuf codec (<code>-api.querier-default-codec</code>)</li></ul></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/contributing/design-patterns-and-code-conventions/index.html b/docs/contributing/design-patterns-and-code-conventions/index.html
index adbb646f6a..d1ad2092ec 100644
--- a/docs/contributing/design-patterns-and-code-conventions/index.html
+++ b/docs/contributing/design-patterns-and-code-conventions/index.html
@@ -4,7 +4,7 @@
 Go coding style Cortex follows the Go Code Review Comments styleguide and the Formatting and style section of Peter Bourgon&rsquo;s Go: Best Practices for Production Environments."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span class=td-sidebar-nav-active-item>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/contributing/design-patterns-and-conventions.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Design%20patterns%20and%20Code%20conventions" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#go-coding-style>Go coding style</a></li><li><a href=#no-global-variables>No global variables</a></li><li><a href=#prometheus-metrics>Prometheus metrics</a></li><li><a href=#config-file-and-cli-flags-conventions>Config file and CLI flags conventions</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/contributing/>Contributing</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/contributing/design-patterns-and-code-conventions/ aria-disabled=true class="btn-link disabled">Design patterns and Code conventions</a></li></ol></nav><div class=td-content><h1>Design patterns and Code conventions</h1><header class=article-meta></header><p>Cortex adopts some design patterns and code conventions that we ask you to follow when contributing to the project. These conventions have been adopted based on the experience gained over the time and aim to enforce good coding practices and keep a consistent UX (ie. config).</p><h2 id=go-coding-style>Go coding style</h2><p>Cortex follows the <a href=https://github.com/golang/go/wiki/CodeReviewComments>Go Code Review Comments</a> styleguide and the <a href=https://peter.bourgon.org/go-in-production/#formatting-and-style>Formatting and style</a> section of Peter Bourgon&rsquo;s <a href=https://peter.bourgon.org/go-in-production/>Go: Best Practices for Production Environments</a>.</p><h2 id=no-global-variables>No global variables</h2><ul><li>Do not use global variables</li></ul><h2 id=prometheus-metrics>Prometheus metrics</h2><p>When registering a metric:</p><ul><li>Do not use a global variable for the metric</li><li>Create and register the metric with <code>promauto.With(reg)</code></li><li>In any internal Cortex component, do not register the metric to the default prometheus registerer, but pick the registerer in input (ie. <code>NewComponent(reg prometheus.Registerer)</code>)</li></ul><p>Testing metrics:</p><ul><li>When writing using tests, test exported metrics using <code>testutil.GatherAndCompare()</code></li></ul><h2 id=config-file-and-cli-flags-conventions>Config file and CLI flags conventions</h2><p>Naming:</p><ul><li>Config file options should be lowercase with words <code>_</code> (underscore) separated (ie. <code>memcached_client</code>)</li><li>CLI flags should be lowercase with words <code>-</code> (dash) separated (ie. <code>memcached-client</code>)</li><li>When adding a new config option, look if a similar one already exists within the <a href=/docs/configuration/configuration-file/>config</a> and keep the same naming (ie. <code>addresses</code> for a list of network endpoints)</li></ul><p>Documentation:</p><ul><li>A CLI flag mentioned in the documentation or changelog should be always prefixed with a single <code>-</code> (dash)</li></ul></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/contributing/governance/index.html b/docs/contributing/governance/index.html
index 88b8156d4d..36631a4ff9 100644
--- a/docs/contributing/governance/index.html
+++ b/docs/contributing/governance/index.html
@@ -9,7 +9,7 @@
 For formal votes, a specific statement of what is being voted on should be added to the relevant github issue or PR, and a link to that issue or PR added to the maintainers meeting agenda document."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span class=td-sidebar-nav-active-item>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/GOVERNANCE.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Governance" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#voting>Voting</a></li><li><a href=#maintainer-role>Maintainer Role</a></li><li><a href=#changes-in-maintainership>Changes in Maintainership</a></li><li><a href=#approving-prs>Approving PRs</a></li><li><a href=#triage-role>Triage Role</a></li><li><a href=#github-project-administration>Github Project Administration</a></li><li><a href=#changes-in-governance>Changes in Governance</a></li><li><a href=#cortex-team>Cortex Team</a><ul><li><a href=#maintainers>Maintainers</a></li><li><a href=#triaggers>Triaggers</a></li></ul></li><li><a href=#other-changes>Other Changes</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/contributing/>Contributing</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/contributing/governance/ aria-disabled=true class="btn-link disabled">Governance</a></li></ol></nav><div class=td-content><h1>Governance</h1><header class=article-meta></header><p>This document defines project governance for the project.</p><h2 id=voting>Voting</h2><p>The Cortex project employs voting to ensure no single member can dominate the project. Any maintainer may cast a vote. To avoid having a single company dominate the project, at most two votes from maintainers working for the same company will count.</p><p>For formal votes, a specific statement of what is being voted on should be added to the relevant github issue or PR, and a link to that issue or PR added to the maintainers meeting agenda document.</p><p>Maintainers should indicate their yes/no vote on that issue or PR, and after a suitable period of time (minimum 2 business weeks), the votes will be tallied and the outcome noted. Maintainers who do not cast a vote, after a suitable period of time, are not included in the majority calculation.</p><h2 id=maintainer-role>Maintainer Role</h2><p>Maintainers are required to participate in the project, by joining discussions, submitting and reviewing pull requests, answering user questions, among others.</p><p>Besides that, we have one concrete activity in which maintainers have to engage from time to time: releasing new versions of Cortex. This process ideally takes only a couple of hours, but requires coordination on different fronts. Even though the process is well documented, it is not without eventual glitches, so, each release needs a &ldquo;Release shepherd&rdquo;. How it works is described in the <a href=https://github.com/cortexproject/cortex/blob/master/RELEASE.md>RELEASE.md</a> file.</p><h2 id=changes-in-maintainership>Changes in Maintainership</h2><p>Contributors who are interested in becoming a maintainer, if performing relevant responsibilities, should discuss their interest with the existing maintainers. New maintainers must be nominated by an existing maintainer and must be elected by 2/3 majority vote.</p><p>We do not expect anyone to make a permanent commitment to be a Cortex maintainer forever. After all, circumstances change,
diff --git a/docs/contributing/how-integration-tests-work/index.html b/docs/contributing/how-integration-tests-work/index.html
index 5614ba148d..453f92bdf0 100644
--- a/docs/contributing/how-integration-tests-work/index.html
+++ b/docs/contributing/how-integration-tests-work/index.html
@@ -4,7 +4,7 @@
 How to run integration tests When integration tests run in CI, we build the Cortex docker image based on the PR code and then run the integration tests against it."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span class=td-sidebar-nav-active-item>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/contributing/how-integration-tests-work.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=How%20integration%20tests%20work" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#how-to-run-integration-tests>How to run integration tests</a><ul><li><a href=#supported-environment-variables>Supported environment variables</a></li><li><a href=#the-requires_docker-tag>The <code>requires_docker</code> tag</a></li></ul></li><li><a href=#isolation>Isolation</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/contributing/>Contributing</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/contributing/how-integration-tests-work/ aria-disabled=true class="btn-link disabled">How integration tests work</a></li></ol></nav><div class=td-content><h1>How integration tests work</h1><header class=article-meta></header><p>Cortex integration tests are written in Go and based on a <a href=https://github.com/cortexproject/cortex/tree/master/integration/e2e>custom framework</a> running Cortex and its dependencies in Docker containers and using the Go <a href=https://golang.org/pkg/testing/><code>testing</code></a> package for assertions. Integration tests run in CI for every PR, and can be easily executed locally during development (it just requires Docker).</p><h2 id=how-to-run-integration-tests>How to run integration tests</h2><p>When integration tests run in CI, we build the Cortex docker image based on the PR code and then run the integration tests against it. When running tests <strong>locally</strong> you should build the Cortex Docker image first:</p><pre tabindex=0><code>make ./cmd/cortex/.uptodate
diff --git a/docs/contributing/how-to-add-a-maintainer/index.html b/docs/contributing/how-to-add-a-maintainer/index.html
index ecd355282e..fd187b976f 100644
--- a/docs/contributing/how-to-add-a-maintainer/index.html
+++ b/docs/contributing/how-to-add-a-maintainer/index.html
@@ -4,7 +4,7 @@
 Submit a PR to add the new member to MAINTAINERS Invite to GitHub organization Invite to cortex-team group Invite to Quay.io repository Invite to Docker Hub organization Submit a PR to CNCF maintainer list for Cortex Invite to CNCF cncf-cortex-maintainers mailing list (via CNCF Service Desk) Invite to Cortex Project Public Calendar Invite to maintainer&rsquo;s Google Docs Add to the Google Analytics property used for the website statistics Invite to credentials vault "><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span class=td-sidebar-nav-active-item>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/contributing/how-to-add-a-maintainer.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=How%20to%20add%20a%20maintainer" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/contributing/>Contributing</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/contributing/how-to-add-a-maintainer/ aria-disabled=true class="btn-link disabled">How to add a maintainer</a></li></ol></nav><div class=td-content><h1>How to add a maintainer</h1><header class=article-meta></header><p>New maintainers are proposed by an existing maintainer and are elected by <a href=/docs/contributing/governance/#voting>majority vote</a>. Once the vote passed, the following steps should be done to add a new member to the maintainers team:</p><ol><li>Submit a PR to add the new member to <code>MAINTAINERS</code></li><li>Invite to <a href=https://github.com/orgs/cortexproject/people>GitHub organization</a></li><li>Invite to <a href=https://groups.google.com/forum/#!forum/cortex-team><code>cortex-team</code> group</a></li><li>Invite to <a href="https://quay.io/organization/cortexproject?tab=teams">Quay.io repository</a></li><li>Invite to <a href=https://hub.docker.com/u/cortexproject>Docker Hub organization</a></li><li>Submit a PR to <a href=http://maintainers.cncf.io>CNCF maintainer list for Cortex</a></li><li>Invite to <a href=https://lists.cncf.io/g/cncf-cortex-maintainers>CNCF <code>cncf-cortex-maintainers</code> mailing list</a> (via <a href=https://servicedesk.cncf.io>CNCF Service Desk</a>)</li><li>Invite to <a href="https://calendar.google.com/calendar/u/0?cid=Y25jZi1jb3J0ZXgtbWFpbnRhaW5lcnNAbGlzdHMuY25jZi5pbw">Cortex Project Public Calendar</a></li><li>Invite to <a href=https://drive.google.com/drive/folders/1aT7-rx4hvYoB3EjvOeX1LHJhgH0Sawzb>maintainer&rsquo;s Google Docs</a></li><li>Add to the Google Analytics property used for the website statistics</li><li>Invite to credentials vault</li></ol></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/contributing/how-to-run-the-website-locally/index.html b/docs/contributing/how-to-run-the-website-locally/index.html
index 70c8c3a4da..a713e16126 100644
--- a/docs/contributing/how-to-run-the-website-locally/index.html
+++ b/docs/contributing/how-to-run-the-website-locally/index.html
@@ -7,7 +7,7 @@
 Install Hugo (extended version) Look for value of HUGO_VERSION in build-image/Dockerfile. Install Node.js v14 or above (alternatively via nvm) Install required Node modules with: cd website && npm install && cd - Install embedmd v1."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span class=td-sidebar-nav-active-item>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/contributing/how-to-run-website-locally.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=How%20to%20run%20the%20website%20locally" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/contributing/>Contributing</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/contributing/how-to-run-the-website-locally/ aria-disabled=true class="btn-link disabled">How to run the website locally</a></li></ol></nav><div class=td-content><h1>How to run the website locally</h1><header class=article-meta></header><p>The Cortex documentation is compiled into a website published at <a href=https://cortexmetrics.io/>cortexmetrics.io</a>. These instructions explain how to run the website locally, in order to have a quick feedback loop while contributing to the documentation or website styling.</p><h2 id=initial-setup>Initial setup</h2><p>The following initial setup is required only once:</p><ol><li>Install <a href=https://gohugo.io/>Hugo</a> (<strong>extended</strong> version)<ul><li>Look for value of <code>HUGO_VERSION</code> in <code>build-image/Dockerfile</code>.</li></ul></li><li>Install <a href=https://nodejs.org/en/>Node.js</a> <code>v14</code> or above (alternatively via <a href=https://github.com/nvm-sh/nvm><code>nvm</code></a>)</li><li>Install required Node modules with:<pre tabindex=0><code>cd website &amp;&amp; npm install &amp;&amp; cd -
diff --git a/docs/contributing/how-to-update-the-build-image/index.html b/docs/contributing/how-to-update-the-build-image/index.html
index fe43f3b9d6..79dcbf7dcd 100644
--- a/docs/contributing/how-to-update-the-build-image/index.html
+++ b/docs/contributing/how-to-update-the-build-image/index.html
@@ -7,7 +7,7 @@
 Update build-image/Dockerfile Create a PR to master with that changed, after the PR is merged to master, the new build image is available in the quay.io repository. Check github action logs here for to find the image tag. Create another PR to replace the image tag in .github/workflows/* (there may be multiple references) and Makefile (variable LATEST_BUILD_IMAGE_TAG). If you are updating Go&rsquo;s runtime version be sure to change actions/setup-go&rsquo;s go-version in ``."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span class=td-sidebar-nav-active-item>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/contributing/how-to-update-the-build-image.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=How%20to%20update%20the%20build%20image" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/contributing/>Contributing</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/contributing/how-to-update-the-build-image/ aria-disabled=true class="btn-link disabled">How to update the build image</a></li></ol></nav><div class=td-content><h1>How to update the build image</h1><header class=article-meta></header><p>The procedure is:</p><ol><li>Update <code>build-image/Dockerfile</code></li><li>Create a PR to master with that changed, after the PR is merged to master, the new build image is available in the quay.io repository. Check github action logs <a href=https://github.com/cortexproject/cortex/actions/workflows/build-image.yml>here</a> for to find the image tag.</li><li>Create another PR to replace the image tag in <code>.github/workflows/*</code> (<em>there may be multiple references</em>) and Makefile (variable <code>LATEST_BUILD_IMAGE_TAG</code>).</li><li>If you are updating Go&rsquo;s runtime version be sure to change <code>actions/setup-go</code>&rsquo;s <code>go-version</code> in ``.github/workflows/*`.</li><li>Open a PR and make sure the CI with new build-image passes</li></ol></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/contributing/how-to-upgrade-golang-version/index.html b/docs/contributing/how-to-upgrade-golang-version/index.html
index 3903e7caea..eeb5340a89 100644
--- a/docs/contributing/how-to-upgrade-golang-version/index.html
+++ b/docs/contributing/how-to-upgrade-golang-version/index.html
@@ -11,7 +11,7 @@
 Upgrade go version in go.mod "><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span class=td-sidebar-nav-active-item>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/contributing/how-to-upgrade-golang-version.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=How%20to%20upgrade%20Golang%20version" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/contributing/>Contributing</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/contributing/how-to-upgrade-golang-version/ aria-disabled=true class="btn-link disabled">How to upgrade Golang version</a></li></ol></nav><div class=td-content><h1>How to upgrade Golang version</h1><header class=article-meta></header><p>To upgrade the Go&rsquo;s runtime version:</p><ol><li>Upgrade build image version with golang version as describe <a href=/docs/contributing/how-to-update-the-build-image/>here</a></li></ol><p>If the minimum support Go&rsquo;s language version should be upgraded as well:</p><ol><li>Upgrade <code>go</code> version in <code>go.mod</code></li></ol></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/contributing/index.html b/docs/contributing/index.html
index 76c1022852..a11c31c499 100644
--- a/docs/contributing/index.html
+++ b/docs/contributing/index.html
@@ -3,7 +3,7 @@
 Cortex follows a standard GitHub …"><meta property="og:title" content="Contributing"><meta property="og:description" content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta property="og:type" content="website"><meta property="og:url" content="/docs/contributing/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Contributing"><meta itemprop=description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Contributing"><meta name=twitter:description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span class=td-sidebar-nav-active-item>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/contributing/_index.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Contributing" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#workflow>Workflow</a></li><li><a href=#formatting>Formatting</a></li><li><a href=#developer-certificates-of-origin-dcos>Developer Certificates of Origin (DCOs)</a></li><li><a href=#building-cortex>Building Cortex</a><ul><li><a href=#dependency-management>Dependency management</a></li></ul></li><li><a href=#design-patterns-and-code-conventions>Design patterns and Code conventions</a></li><li><a href=#documentation>Documentation</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/contributing/ aria-disabled=true class="btn-link disabled">Contributing</a></li></ol></nav><div class=td-content><h1>Contributing</h1><header class=article-meta></header><p>Welcome! We&rsquo;re excited that you&rsquo;re interested in contributing. Below are some basic guidelines.</p><h2 id=workflow>Workflow</h2><p>Cortex follows a standard GitHub pull request workflow. If you&rsquo;re unfamiliar with this workflow, read the very helpful <a href=https://guides.github.com/introduction/flow/>Understanding the GitHub flow</a> guide from GitHub.</p><p>You are welcome to create draft PRs at any stage of readiness - this
diff --git a/docs/getting-started/index.html b/docs/getting-started/index.html
index 0c16878ec7..c0aa9df88b 100644
--- a/docs/getting-started/index.html
+++ b/docs/getting-started/index.html
@@ -3,7 +3,7 @@
 There are two guides …"><meta property="og:title" content="Getting Started"><meta property="og:description" content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta property="og:type" content="website"><meta property="og:url" content="/docs/getting-started/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Getting Started"><meta itemprop=description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Getting Started"><meta name=twitter:description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span class=td-sidebar-nav-active-item>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/getting-started/_index.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Getting%20Started" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#single-binary-mode>Single Binary Mode</a><ul><li><a href=#prerequisites>Prerequisites</a></li><li><a href=#running-cortex-as-a-single-instance>Running Cortex as a Single Instance</a></li><li><a href=#clean-up>Clean up</a></li></ul></li><li><a href=#microservice-mode>Microservice Mode</a><ul><li><a href=#prerequisites-1>Prerequisites</a></li><li><a href=#setup-kind>Setup Kind</a></li><li><a href=#configure-helm>Configure Helm</a></li><li><a href=#instructions-1>Instructions</a></li><li><a href=#clean-up-1>Clean up</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/getting-started/ aria-disabled=true class="btn-link disabled">Getting Started</a></li></ol></nav><div class=td-content><h1>Getting Started</h1><header class=article-meta></header><p>Cortex is a powerful platform software that can be run in two modes: as a single binary or as multiple
diff --git a/docs/guides/alertmanager-configuration/index.html b/docs/guides/alertmanager-configuration/index.html
index 339a2b016a..974e52cea8 100644
--- a/docs/guides/alertmanager-configuration/index.html
+++ b/docs/guides/alertmanager-configuration/index.html
@@ -5,7 +5,7 @@
 Configuring the Cortex Alertmanager storage backend With the introduction of Cortex 1.8, the storage backend config option shifted to the new pattern #3888. You can find the new configuration here"><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span class=td-sidebar-nav-active-item>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/alert-manager-configuration.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Configuring%20Notification%20using%20Cortex%20Alertmanager" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#context>Context</a><ul><li><a href=#configuring-the-cortex-alertmanager-storage-backend>Configuring the Cortex Alertmanager storage backend</a></li><li><a href=#cortex-alertmanager-configuration>Cortex Alertmanager configuration</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/alertmanager-configuration/ aria-disabled=true class="btn-link disabled">Alertmanager configuration</a></li></ol></nav><div class=td-content><h1>Configuring Notification using Cortex Alertmanager</h1><header class=article-meta></header><h2 id=context>Context</h2><p>Cortex Alertmanager notification setup follows mostly the syntax of Prometheus Alertmanager since it is based on the same codebase. The following is a description on how to load the configuration setup so that Alertmanager can use it for notification when an alert event happens.</p><h3 id=configuring-the-cortex-alertmanager-storage-backend>Configuring the Cortex Alertmanager storage backend</h3><p>With the introduction of Cortex 1.8, the storage backend config option shifted to the new pattern <a href=https://github.com/cortexproject/cortex/pull/3888>#3888</a>. You can find the new configuration <a href=/docs/configuration/configuration-file/#alertmanager_storage_config>here</a></p><p>Note that when using <code>-alertmanager.sharding-enabled=true</code>, the following storage backends are not supported: <code>local</code>, <code>configdb</code>.</p><p>When using the new configuration pattern, it is important that any of the old configuration pattern flags are unset (<code>-alertmanager.storage</code>), as well as <code>-&lt;prefix>.configs.url</code>. This is because the old pattern still takes precedence over the new one. The old configuration pattern (<code>-alertmanager.storage</code>) is marked as deprecated and will be removed by Cortex version 1.11. However, this change doesn&rsquo;t apply to <code>-alertmanager.storage.path</code> and <code>-alertmanager.storage.retention</code>.</p><h3 id=cortex-alertmanager-configuration>Cortex Alertmanager configuration</h3><p>Cortex Alertmanager can be uploaded via Cortex <a href=/docs/api/#set-alertmanager-configuration>Set Alertmanager configuration API</a> or using <a href=https://github.com/cortexproject/cortex-tools>Cortex Tools</a>.</p><p>Follow the instructions at the <code>cortextool</code> link above to download or update to the latest version of the tool.</p><p>To obtain the full help of how to use <code>cortextool</code> for all commands and flags, use
diff --git a/docs/guides/auth/index.html b/docs/guides/auth/index.html
index f999240a88..f63db6570b 100644
--- a/docs/guides/auth/index.html
+++ b/docs/guides/auth/index.html
@@ -5,7 +5,7 @@
 Typically, this means you run Cortex behind a reverse proxy, and you must ensure that all callers, both machines sending data over the remote_write interface and humans sending queries from GUIs, supply credentials which identify them and confirm they are authorised."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span class=td-sidebar-nav-active-item>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/authentication-and-authorisation.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Authentication%20and%20Authorisation" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/auth/ aria-disabled=true class="btn-link disabled">Authentication and Authorisation</a></li></ol></nav><div class=td-content><h1>Authentication and Authorisation</h1><header class=article-meta></header><p>All Cortex components take the tenant ID from a header <code>X-Scope-OrgID</code>
diff --git a/docs/guides/capacity-planning/index.html b/docs/guides/capacity-planning/index.html
index d817d7e075..709d3fbe03 100644
--- a/docs/guides/capacity-planning/index.html
+++ b/docs/guides/capacity-planning/index.html
@@ -12,7 +12,7 @@
 The number of active series. If you have Prometheus already, you can query prometheus_tsdb_head_series to see this number."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span class=td-sidebar-nav-active-item>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/capacity-planning.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Capacity%20Planning" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/capacity-planning/ aria-disabled=true class="btn-link disabled">Capacity Planning</a></li></ol></nav><div class=td-content><h1>Capacity Planning</h1><header class=article-meta></header><p><em>This doc is likely out of date. It should be updated for blocks storage.</em></p><p>You will want to estimate how many nodes are required, how many of
diff --git a/docs/guides/encryption-at-rest/index.html b/docs/guides/encryption-at-rest/index.html
index 08372158db..e9e655be22 100644
--- a/docs/guides/encryption-at-rest/index.html
+++ b/docs/guides/encryption-at-rest/index.html
@@ -15,7 +15,7 @@
 sse: # Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3. # CLI flag: -<prefix>.s3.sse.type [type: <string> | default = &#34;&#34;] # KMS Key ID used to encrypt objects in S3 # CLI flag: -<prefix>."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span class=td-sidebar-nav-active-item>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/encryption-at-rest.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Encryption%20at%20Rest" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#s3>S3</a><ul><li><a href=#blocks-storage>Blocks storage</a></li><li><a href=#s3_sse_config><code>s3_sse_config</code></a></li><li><a href=#ruler>Ruler</a></li><li><a href=#alertmanager>Alertmanager</a></li><li><a href=#per-tenant-config-overrides>Per-tenant config overrides</a></li></ul></li><li><a href=#other-storages>Other storages</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/encryption-at-rest/ aria-disabled=true class="btn-link disabled">Encryption at Rest</a></li></ol></nav><div class=td-content><h1>Encryption at Rest</h1><header class=article-meta></header><p>Cortex supports data encryption at rest for some storage backends.</p><h2 id=s3>S3</h2><p>The Cortex S3 client supports the following server-side encryption (SSE) modes:</p><ul><li><a href=https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html>SSE-S3</a></li><li><a href=https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html>SSE-KMS</a></li></ul><h3 id=blocks-storage>Blocks storage</h3><p>The <a href=/docs/blocks-storage/>blocks storage</a> S3 server-side encryption can be configured as follows.</p><h3 id=s3_sse_config><code>s3_sse_config</code></h3><p>The <code>s3_sse_config</code> configures the S3 server-side encryption.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-yaml data-lang=yaml><span style=display:flex><span><span style=color:#204a87;font-weight:700>sse</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline>
diff --git a/docs/guides/getting-started-with-gossiped-ring/index.html b/docs/guides/getting-started-with-gossiped-ring/index.html
index 2e87e278a8..a6f55c2f3e 100644
--- a/docs/guides/getting-started-with-gossiped-ring/index.html
+++ b/docs/guides/getting-started-with-gossiped-ring/index.html
@@ -5,7 +5,7 @@
 This short guide shows how to start Cortex in single-binary mode with a memberlist-based ring. To reduce the number of required dependencies in this guide, it will use blocks storage with no shipping to external stores."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span class=td-sidebar-nav-active-item>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/gossip-ring-getting-started.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Getting%20started%20with%20gossiped%20ring" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#single-binary-two-cortex-instances>Single-binary, two Cortex instances</a></li><li><a href=#how-it-works>How it works</a></li><li><a href=#how-to-add-another-instance>How to add another instance?</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/getting-started-with-gossiped-ring/ aria-disabled=true class="btn-link disabled">Getting started with a gossip ring cluster</a></li></ol></nav><div class=td-content><h1>Getting started with gossiped ring</h1><header class=article-meta></header><p>Cortex requires a Key-Value (KV) store to store the ring. It can use traditional KV stores like Consul or Etcd,
diff --git a/docs/guides/glossary/index.html b/docs/guides/glossary/index.html
index 61ad3b08e5..5c069395c3 100644
--- a/docs/guides/glossary/index.html
+++ b/docs/guides/glossary/index.html
@@ -17,7 +17,7 @@
 A series becomes idle once it&rsquo;s not exported anymore by the monitored targets."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span class=td-sidebar-nav-active-item>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/glossary.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Glossary" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#blocks-storage>Blocks storage</a></li><li><a href=#chunk>Chunk</a></li><li><a href=#churn>Churn</a></li><li><a href=#flushing>Flushing</a></li><li><a href=#ha-tracker>HA Tracker</a></li><li><a href=#hash-ring>Hash ring</a></li><li><a href=#org>Org</a></li><li><a href=#ring>Ring</a></li><li><a href=#sample>Sample</a></li><li><a href=#series>Series</a></li><li><a href=#tenant>Tenant</a></li><li><a href=#time-series>Time series</a></li><li><a href=#user>User</a></li><li><a href=#wal>WAL</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/glossary/ aria-disabled=true class="btn-link disabled">Glossary</a></li></ol></nav><div class=td-content><h1>Glossary</h1><header class=article-meta></header><h3 id=blocks-storage>Blocks storage</h3><p>The blocks storage is a Cortex storage engine based on Prometheus TSDB, which only requires an object store (eg. AWS S3, Google GCS, &mldr;) as backend storage.</p><p>For more information, please refer to the <a href=/docs/blocks-storage/>Cortex blocks storage</a> documentation.</p><h3 id=chunk>Chunk</h3><p>A chunk is an object containing compressed timestamp-value pairs.</p><p>A single chunk contains timestamp-value pairs for several series.</p><h3 id=churn>Churn</h3><p>Churn is the frequency at which series become idle.</p><p>A series becomes idle once it&rsquo;s not exported anymore by the monitored targets. Typically, series become idle when the monitored target itself disappears (eg. the process or node gets terminated).</p><h3 id=flushing>Flushing</h3><p>Series flushing is the operation run by ingesters to offload time series from memory and store them in the long-term storage.</p><h3 id=ha-tracker>HA Tracker</h3><p>The HA Tracker is a feature of Cortex distributor which is used to deduplicate received series coming from two (or more) Prometheus servers configured in HA pairs.</p><p>For more information, please refer to the guide &ldquo;<a href=/docs/guides/ha-pair-handling/>Config for sending HA Pairs data to Cortex</a>&rdquo;.</p><h3 id=hash-ring>Hash ring</h3><p>The hash ring is a distributed data structure used by Cortex for sharding, replication, and service discovery. The hash ring data structure gets shared across Cortex replicas via gossip or a key-value store.</p><p>For more information, please refer to the <a href=/docs/architecture/#the-hash-ring>Architecture</a> documentation.</p><h3 id=org>Org</h3><p><em>See <a href=#tenant>Tenant</a>.</em></p><h3 id=ring>Ring</h3><p><em>See <a href=#hash-ring>Hash ring</a>.</em></p><h3 id=sample>Sample</h3><p>A sample is a single timestamped value in a time series.</p><p>For example, given the series <code>node_cpu_seconds_total{instance="10.0.0.1",mode="system"}</code> its stream of values (samples) could be:</p><pre tabindex=0><code># Display format: &lt;value&gt; @&lt;timestamp&gt;
diff --git a/docs/guides/ha-pair-handling/index.html b/docs/guides/ha-pair-handling/index.html
index be47d9298c..11a79b2424 100644
--- a/docs/guides/ha-pair-handling/index.html
+++ b/docs/guides/ha-pair-handling/index.html
@@ -5,7 +5,7 @@
 Assume that there are two teams, each running their own Prometheus, monitoring different services."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span class=td-sidebar-nav-active-item>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/ha-pair-handling.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Config%20for%20sending%20HA%20Pairs%20data%20to%20Cortex" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#context>Context</a></li><li><a href=#config>Config</a><ul><li><a href=#client-side>Client Side</a></li><li><a href=#server-side>Server Side</a></li></ul></li><li><a href=#remote-read>Remote Read</a></li><li><a href=#accept-multiple-ha-pairs-in-single-request>Accept multiple HA pairs in single request</a></li><li><a href=#config-1>Config</a><ul><li><a href=#client-side-1>Client side</a></li><li><a href=#server-side-1>Server side</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/ha-pair-handling/ aria-disabled=true class="btn-link disabled">Config for sending HA Pairs data to Cortex</a></li></ol></nav><div class=td-content><h1>Config for sending HA Pairs data to Cortex</h1><header class=article-meta></header><h2 id=context>Context</h2><p>You can have more than a single Prometheus monitoring and ingesting the same metrics for redundancy. Cortex already does replication for redundancy, and it doesn&rsquo;t make sense to ingest the same data twice. So in Cortex, we made sure we can dedupe the data we receive from HA Pairs of Prometheus. We do this via the following:</p><p>Assume that there are two teams, each running their own Prometheus, monitoring different services. Let&rsquo;s call the Prometheus T1 and T2. Now, if the teams are running HA pairs, let&rsquo;s call the individual Prometheus, T1.a, T1.b, and T2.a and T2.b.</p><p>In Cortex, we make sure we only ingest from one of T1.a and T1.b, and only from one of T2.a and T2.b. We do this by electing a leader replica for each cluster of Prometheus. For example, in the case of T1, let it be T1.a. As long as T1.a is the leader, we drop the samples sent by T1.b. And if Cortex sees no new samples from T1.a for a short period (30s by default), it&rsquo;ll switch the leader to be T1.b.</p><p>This means if T1.a goes down for a few minutes, Cortex&rsquo;s HA sample handling will have switched and elected T1.b as the leader. This failover timeout is what enables us to only accept samples from a single replica at a time, but ensure we don&rsquo;t drop too much data in case of issues. Note that with the default scrape period of 15s, and the default timeouts in Cortex, in most cases, you&rsquo;ll only lose a single scrape of data in the case of a leader election failover. For any rate queries, the rate window should be at least 4x the scrape period to account for any of these failover scenarios, for example, with the default scrape period of 15s, then you should calculate rates over at least 1m periods.</p><p>Now we do the same leader election process for T2.</p><h2 id=config>Config</h2><h3 id=client-side>Client Side</h3><p>So for Cortex to achieve this, we need 2 identifiers for each process, one identifier for the cluster (T1 or T2, etc.) and one identifier to identify the replica in the cluster (a or b). The easiest way to do this is by setting external labels; the default labels are <code>cluster</code> and <code>__replica__</code>. For example:</p><pre tabindex=0><code>cluster: prom-team1
diff --git a/docs/guides/index.html b/docs/guides/index.html
index 53961192bc..035e11ae0b 100644
--- a/docs/guides/index.html
+++ b/docs/guides/index.html
@@ -1,7 +1,7 @@
 <!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.101.0"><link rel=alternate type=application/rss+xml href=/docs/guides/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/favicons/android-96x96.png sizes=96x96><link rel=icon type=image/png href=/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/favicons/android-192x192.png sizes=192x192><title>Guides | Cortex</title><meta name=description content><meta property="og:title" content="Guides"><meta property="og:description" content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta property="og:type" content="website"><meta property="og:url" content="/docs/guides/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Guides"><meta itemprop=description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Guides"><meta name=twitter:description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__section" id=m-docsguides><span class=td-sidebar-nav-active-item>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/_index.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Guides" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/ aria-disabled=true class="btn-link disabled">Guides</a></li></ol></nav><div class=td-content><h1>Guides</h1><header class=article-meta></header><ul><li><a href=/docs/guides/running-cortex-on-kubernetes/>Running Cortex on Kubernetes</a></li><li><a href=/docs/guides/getting-started-with-gossiped-ring/>Getting started with gossiped ring</a></li><li><a href=/docs/guides/alertmanager-configuration/>Configuring Notification using Cortex Alertmanager</a></li><li><a href=/docs/guides/auth/>Authentication and Authorisation</a></li><li><a href=/docs/guides/capacity-planning/>Capacity Planning</a></li><li><a href=/docs/guides/ruler-sharding/>Config for horizontally scaling the Ruler</a></li><li><a href=/docs/guides/ha-pair-handling/>Config for sending HA Pairs data to Cortex</a></li><li><a href=/docs/guides/encryption-at-rest/>Encryption at Rest</a></li><li><a href=/docs/guides/ingesters-rolling-updates/>Ingesters rolling updates</a></li><li><a href=/docs/guides/ingesters-scaling-up-and-down/>Ingesters scaling up and down</a></li><li><a href=/docs/guides/overrides-exporter/>Overrides Exporter</a></li><li><a href=/docs/guides/tls/>Securing communication between Cortex components with TLS</a></li><li><a href=/docs/guides/security/>Security</a></li><li><a href=/docs/guides/shuffle-sharding/>Shuffle Sharding</a></li><li><a href=/docs/guides/tracing/>Tracing</a></li><li><a href=/docs/guides/zone-aware-replication/>Zone Aware Replication</a></li><li><a href=/docs/guides/limitations/>Limitations</a></li><li><a href=/docs/guides/glossary/>Glossary</a></li></ul></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/guides/ingesters-rolling-updates/index.html b/docs/guides/ingesters-rolling-updates/index.html
index a9f8e7d9a4..97fef3df66 100644
--- a/docs/guides/ingesters-rolling-updates/index.html
+++ b/docs/guides/ingesters-rolling-updates/index.html
@@ -5,7 +5,7 @@
 The Cortex blocks storage requires ingesters to run with a persistent disk where the TSDB WAL and blocks are stored (eg. a StatefulSet when deployed on Kubernetes)."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span class=td-sidebar-nav-active-item>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/ingesters-rolling-updates.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Ingesters%20rolling%20updates" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/ingesters-rolling-updates/ aria-disabled=true class="btn-link disabled">Ingesters rolling updates</a></li></ol></nav><div class=td-content><h1>Ingesters rolling updates</h1><header class=article-meta></header><p>Cortex <a href=/docs/architecture/#ingester>ingesters</a> are semi-stateful.
diff --git a/docs/guides/ingesters-scaling-up-and-down/index.html b/docs/guides/ingesters-scaling-up-and-down/index.html
index db72cf6911..13655cf14e 100644
--- a/docs/guides/ingesters-scaling-up-and-down/index.html
+++ b/docs/guides/ingesters-scaling-up-and-down/index.html
@@ -8,7 +8,7 @@
 Scaling up Adding more ingesters to a Cortex cluster is considered a safe operation. When a new ingester starts, it will register to the hash ring and the distributors will reshard received series accordingly. Ingesters that were previously receiving those series will see data stop arriving and will consider those series &ldquo;idle&rdquo;."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span class=td-sidebar-nav-active-item>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/ingesters-scaling-up-and-down.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Ingesters%20scaling%20up%20and%20down" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/ingesters-scaling-up-and-down/ aria-disabled=true class="btn-link disabled">Ingesters scaling up and down</a></li></ol></nav><div class=td-content><h1>Ingesters scaling up and down</h1><header class=article-meta></header><p>This guide explains how to scale up and down ingesters.</p><p>If you&rsquo;re looking for how to run ingesters rolling updates, please refer to the <a href=/docs/guides/ingesters-rolling-updates/>dedicated guide</a>._</p><h2 id=scaling-up>Scaling up</h2><p>Adding more ingesters to a Cortex cluster is considered a safe operation. When a new ingester starts, it will register to the <a href=/docs/architecture/#the-hash-ring>hash ring</a> and the distributors will reshard received series accordingly.
diff --git a/docs/guides/limitations/index.html b/docs/guides/limitations/index.html
index 6d662ef5c0..8204d4fa75 100644
--- a/docs/guides/limitations/index.html
+++ b/docs/guides/limitations/index.html
@@ -8,7 +8,7 @@
 Alphanumeric characters 0-9 a-z A-Z Special characters Exclamation point (!"><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span class=td-sidebar-nav-active-item>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/limitations.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Limitations" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#tenant-id-naming>Tenant ID naming</a><ul><li><a href=#supported-characters>Supported characters</a></li><li><a href=#length>Length</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/limitations/ aria-disabled=true class="btn-link disabled">Limitations</a></li></ol></nav><div class=td-content><h1>Limitations</h1><header class=article-meta></header><h2 id=tenant-id-naming>Tenant ID naming</h2><p>The tenant ID (also called &ldquo;user ID&rdquo; or &ldquo;org ID&rdquo;) is the unique identifier of a tenant within a Cortex cluster. The tenant ID is opaque information to Cortex, which doesn&rsquo;t make any assumptions on its format/content, but its naming has two limitations:</p><ol><li>Supported characters</li><li>Length</li></ol><h3 id=supported-characters>Supported characters</h3><p>The following character sets are generally <strong>safe for use in the tenant ID</strong>:</p><ul><li>Alphanumeric characters<ul><li><code>0-9</code></li><li><code>a-z</code></li><li><code>A-Z</code></li></ul></li><li>Special characters<ul><li>Exclamation point (<code>!</code>)</li><li>Hyphen (<code>-</code>)</li><li>Underscore (<code>_</code>)</li><li>Single Period (<code>.</code>), but the tenant IDs <code>.</code> and <code>..</code> are considered invalid</li><li>Asterisk (<code>*</code>)</li><li>Single quote (<code>'</code>)</li><li>Open parenthesis (<code>(</code>)</li><li>Close parenthesis (<code>)</code>)</li></ul></li></ul><p>All other characters are not safe to use. In particular, slashes <code>/</code> and whitespaces (<code> </code>) are <strong>not supported</strong>.</p><h3 id=length>Length</h3><p>The tenant ID length should not exceed 150 bytes/characters.</p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/guides/overrides-exporter/index.html b/docs/guides/overrides-exporter/index.html
index 3d55be5c87..fee03c9218 100644
--- a/docs/guides/overrides-exporter/index.html
+++ b/docs/guides/overrides-exporter/index.html
@@ -5,7 +5,7 @@
 Context To update configuration without restarting, Cortex allows operators to supply a runtime_config file that will be periodically reloaded. This file can be specified under the runtime_config section of the main configuration file or using the -runtime-config."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span class=td-sidebar-nav-active-item>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/overrides-exporter.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Overrides%20Exporter" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/overrides-exporter/ aria-disabled=true class="btn-link disabled">Overrides Exporter</a></li></ol></nav><div class=td-content><h1>Overrides Exporter</h1><header class=article-meta></header><p>Since Cortex is a multi-tenant system, it supports applying limits to each tenant to prevent
diff --git a/docs/guides/ruler-sharding/index.html b/docs/guides/ruler-sharding/index.html
index 8d1e0a3a7d..66788317bf 100644
--- a/docs/guides/ruler-sharding/index.html
+++ b/docs/guides/ruler-sharding/index.html
@@ -8,7 +8,7 @@
 -ruler.enable-sharding=true In addition, the ruler requires its own ring to be configured, for instance:"><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span class=td-sidebar-nav-active-item>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/sharded_ruler.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Config%20for%20horizontally%20scaling%20the%20Ruler" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/ruler-sharding/ aria-disabled=true class="btn-link disabled">Config for horizontally scaling the Ruler</a></li></ol></nav><div class=td-content><h1>Config for horizontally scaling the Ruler</h1><header class=article-meta></header><h2 id=context>Context</h2><p>One option to scale the ruler is by scaling it horizontally. However, with multiple ruler instances running, they will need to coordinate to determine which instance will evaluate which rule. Similar to the ingesters, the rulers establish a hash ring to divide up the responsibilities of evaluating rules.</p><h2 id=config>Config</h2><p>In order to enable sharding in the ruler, the following flag needs to be set:</p><pre tabindex=0><code>  -ruler.enable-sharding=true
diff --git a/docs/guides/running-cortex-on-kubernetes/index.html b/docs/guides/running-cortex-on-kubernetes/index.html
index 7d460233fc..84a4d5cdd5 100644
--- a/docs/guides/running-cortex-on-kubernetes/index.html
+++ b/docs/guides/running-cortex-on-kubernetes/index.html
@@ -12,7 +12,7 @@
 resources: requests: cpu: 4 memory: 10Gi The specific values here should be adjusted based on your own experiences running Cortex - they are very dependent on the rate of data arriving and other factors such as series churn."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span class=td-sidebar-nav-active-item>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/running-cortex-on-kubernetes.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Running%20Cortex%20on%20Kubernetes" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#configuration>Configuration</a><ul><li><a href=#resource-requests>Resource requests</a></li><li><a href=#take-extra-care-with-ingesters>Take extra care with ingesters</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/running-cortex-on-kubernetes/ aria-disabled=true class="btn-link disabled">Running Cortex on Kubernetes</a></li></ol></nav><div class=td-content><h1>Running Cortex on Kubernetes</h1><header class=article-meta></header><p>Because Cortex is designed to run multiple instances of each component
diff --git a/docs/guides/security/index.html b/docs/guides/security/index.html
index 57c05e6ccf..b7ad901ac6 100644
--- a/docs/guides/security/index.html
+++ b/docs/guides/security/index.html
@@ -7,7 +7,7 @@
 Information about security disclosures and mailing lists is in the main repo"><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span class=td-sidebar-nav-active-item>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/security.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Security" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/security/ aria-disabled=true class="btn-link disabled">Security</a></li></ol></nav><div class=td-content><h1>Security</h1><header class=article-meta></header><p>Cortex must be deployed with due care over system configuration, using principles such as &ldquo;least privilege&rdquo; to limit any exposure due to flaws in the source code.</p><p>You must configure authorisation and authentication externally to Cortex; see <a href=/docs/guides/auth/>this guide</a></p><p>Information about security disclosures and mailing lists is <a href=https://github.com/cortexproject/cortex/blob/master/SECURITY.md>in the main repo</a></p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/guides/shuffle-sharding/index.html b/docs/guides/shuffle-sharding/index.html
index 4fb3e9c155..fdf24649f3 100644
--- a/docs/guides/shuffle-sharding/index.html
+++ b/docs/guides/shuffle-sharding/index.html
@@ -6,7 +6,7 @@
 Background The default sharding strategy employed by Cortex distributes the workload across the entire pool of instances running a given service (eg. ingesters). For example, on the write path, each tenant&rsquo;s series are sharded across all ingesters, regardless of how many active series the tenant has or how many different tenants are in the cluster."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span class=td-sidebar-nav-active-item>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/shuffle-sharding.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Shuffle%20Sharding" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#background>Background</a></li><li><a href=#what-is-shuffle-sharding>What is shuffle sharding?</a><ul><li><a href=#low-overlapping-instances-probability>Low overlapping instances probability</a></li></ul></li><li><a href=#cortex-shuffle-sharding>Cortex shuffle sharding</a><ul><li><a href=#guaranteed-properties>Guaranteed properties</a></li><li><a href=#ingesters-shuffle-sharding>Ingesters shuffle sharding</a></li><li><a href=#query-frontend-and-query-scheduler-shuffle-sharding>Query-frontend and Query-scheduler shuffle sharding</a></li><li><a href=#store-gateway-shuffle-sharding>Store-gateway shuffle sharding</a></li><li><a href=#ruler-shuffle-sharding>Ruler shuffle sharding</a></li><li><a href=#compactor-shuffle-sharding>Compactor shuffle sharding</a></li></ul></li><li><a href=#faq>FAQ</a><ul><li><a href=#does-shuffle-sharding-add-additional-overhead-to-the-kv-store>Does shuffle sharding add additional overhead to the KV store?</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/shuffle-sharding/ aria-disabled=true class="btn-link disabled">Shuffle Sharding</a></li></ol></nav><div class=td-content><h1>Shuffle Sharding</h1><header class=article-meta></header><p>Cortex leverages sharding techniques to horizontally scale both single and multi-tenant clusters beyond the capacity of a single node.</p><h2 id=background>Background</h2><p>The <strong>default sharding strategy</strong> employed by Cortex distributes the workload across the entire pool of instances running a given service (eg. ingesters). For example, on the write path, each tenant&rsquo;s series are sharded across all ingesters, regardless of how many active series the tenant has or how many different tenants are in the cluster.</p><p>The default strategy allows for a fair balance on the resources consumed by each instance (ie. CPU and memory) and to maximise these resources across the cluster.</p><p>However, in a <strong>multi-tenant</strong> cluster, this approach also introduces some <strong>downsides</strong>:</p><ol><li>An outage affects all tenants</li><li></li><li>A misbehaving tenant (eg. causing out-of-memory) could affect all other tenants.</li></ol><p>The goal of <strong>shuffle sharding</strong> is to provide an alternative sharding strategy to reduce the blast radius of an outage and better isolate tenants.</p><h2 id=what-is-shuffle-sharding>What is shuffle sharding?</h2><p>Shuffle sharding is a technique used to isolate different tenants’ workloads and to give each tenant a single-tenant experience even if they&rsquo;re running in a shared cluster. This technique has been publicly shared and clearly explained by AWS in their <a href=https://aws.amazon.com/builders-library/workload-isolation-using-shuffle-sharding/>builders&rsquo; library</a> and a reference implementation has been shown in the <a href=https://github.com/awslabs/route53-infima/blob/master/src/main/java/com/amazonaws/services/route53/infima/SimpleSignatureShuffleSharder.java>Route53 Infima library</a>.</p><p>The idea is to assign each tenant a shard composed of a subset of the Cortex service instances, aiming to minimize the overlapping instances between two different tenants. Shuffle sharding brings the following <strong>benefits</strong> over the default sharding strategy:</p><ul><li>An outage on some Cortex cluster instances/nodes will only affect a subset of tenants.</li><li>A misbehaving tenant will affect only its shard instances. Due to the low overlap of instances between different tenants, it&rsquo;s statistically quite likely that any other tenant will run on different instances or only a subset of instances will match the affected ones.</li></ul><p>Shuffle sharding requires no more resources than the default sharding strategy, but instances may be less evenly balanced from time to time.</p><h3 id=low-overlapping-instances-probability>Low overlapping instances probability</h3><p>For example, given a Cortex cluster running <strong>50 ingesters</strong> and assigning <strong>each tenant 4</strong> out of 50 ingesters, shuffling instances between each tenant, we get <strong>230K possible combinations</strong>.</p><p>Randomly picking two different tenants, we have the:</p><ul><li>71% chance that they will not share any instance</li><li>26% chance that they will share only 1 instance</li><li>2.7% chance that they will share 2 instances</li><li>0.08% chance that they will share 3 instances</li><li>Only a 0.0004% chance that their instances will fully overlap</li></ul><p><img src=/images/guides/shuffle-sharding-probability.png alt="Shuffle sharding probability"></p><h2 id=cortex-shuffle-sharding>Cortex shuffle sharding</h2><p>Cortex currently supports shuffle sharding in the following services:</p><ul><li><a href=#ingesters-shuffle-sharding>Ingesters</a></li><li><a href=#query-frontend-and-query-scheduler-shuffle-sharding>Query-frontend / Query-scheduler</a></li><li><a href=#store-gateway-shuffle-sharding>Store-gateway</a></li><li><a href=#ruler-shuffle-sharding>Ruler</a></li><li><a href=#compactor-shuffle-sharding>Compactor</a></li></ul><p>Shuffle sharding is <strong>disabled by default</strong> and needs to be explicitly enabled in the configuration.</p><h3 id=guaranteed-properties>Guaranteed properties</h3><p>The Cortex shuffle sharding implementation guarantees the following properties:</p><ul><li><strong>Stability</strong><br>Given a consistent state of the hash ring, the shuffle sharding algorithm always selects the same instances for a given tenant, even across different machines.</li><li><strong>Consistency</strong><br>Adding or removing 1 instance from the hash ring leads to only 1 instance changed at most, in each tenant&rsquo;s shard.</li><li><strong>Shuffling</strong><br>Probabilistically and for a large enough cluster, it ensures that every tenant gets a different set of instances, with a reduced number of overlapping instances between two tenants to improve failure isolation.</li><li><strong>Zone-awareness</strong><br>When <a href=/docs/guides/zone-aware-replication/>zone-aware replication</a> is enabled, the subset of instances selected for each tenant contains a balanced number of instances for each availability zone.</li></ul><h3 id=ingesters-shuffle-sharding>Ingesters shuffle sharding</h3><p>By default, the Cortex distributor spreads the received series across all running ingesters.</p><p>When shuffle sharding is <strong>enabled</strong> for the ingesters, the distributor and ruler on the <strong>write path</strong> spread each tenant series across <code>-distributor.ingestion-tenant-shard-size</code> number of ingesters, while on the <strong>read path</strong>, the querier and ruler queries only the subset of ingesters holding the series for a given tenant.</p><p><em>The shard size can be overridden on a per-tenant basis in the limits overrides configuration.</em></p><h4 id=ingesters-write-path>Ingesters write path</h4><p>To enable shuffle-sharding for ingesters on the write path, you need to configure the following CLI flags (or their respective YAML config options) to <strong>distributor</strong>, <strong>ingester</strong>, and <strong>ruler</strong>:</p><ul><li><code>-distributor.sharding-strategy=shuffle-sharding</code></li><li><code>-distributor.ingestion-tenant-shard-size=&lt;size></code><br><code>&lt;size></code> should be set to the number of ingesters each tenant series should be sharded to. If <code>&lt;size></code> is greater than the number of available ingesters in the Cortex cluster, the tenant series are sharded across all ingesters.</li></ul><h4 id=ingesters-read-path>Ingesters read path</h4><p>Assuming shuffle-sharding has been enabled for the write path, to enable shuffle-sharding for ingesters on the read path too, you need to configure the following CLI flags (or their respective YAML config options) to <strong>querier</strong> and <strong>ruler</strong>:</p><ul><li><code>-distributor.sharding-strategy=shuffle-sharding</code></li><li><code>-distributor.ingestion-tenant-shard-size=&lt;size></code></li><li><code>-querier.shuffle-sharding-ingesters-lookback-period=&lt;period></code><br>Queriers and rulers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since &rsquo;now - lookback period&rsquo;. The configured lookback <code>&lt;period></code> should be greater than or equal to <code>-querier.query-store-after</code> and <code>-querier.query-ingesters-within</code> if set, and greater than the estimated minimum time it takes for the oldest samples stored in a block uploaded by ingester to be discovered and available for querying (3h with the default configuration).</li></ul><h4 id=rollout-strategy>Rollout strategy</h4><p>If you&rsquo;re running a Cortex cluster with shuffle-sharding disabled and you want to enable it for ingesters, the following rollout strategy should be used to avoid missing querying any time-series in the ingesters’ memory:</p><ol><li>Enable ingesters shuffle-sharding on the <strong>write path</strong></li><li><strong>Wait</strong> at least <code>-querier.shuffle-sharding-ingesters-lookback-period</code> time</li><li>Enable ingesters shuffle-sharding on the <strong>read path</strong></li></ol><h4 id=limitation-decreasing-the-tenant-shard-size>Limitation: decreasing the tenant shard size</h4><p>The current shuffle-sharding implementation in Cortex has a limitation which prevents safely decreasing the tenant shard size if the ingesters shuffle-sharding is enabled on the read path.</p><p>The problem is that if a tenant’s subring decreases in size, there is currently no way for the queriers and rulers to know how big the tenant subring was previously, and hence they will potentially miss an ingester with data for that tenant. In other words, the lookback mechanism to select the ingesters which may have received series since &rsquo;now - lookback period&rsquo; doesn&rsquo;t work correctly if the tenant shard size is decreased.</p><p>This is deemed an infrequent operation that we considered banning, but a workaround still exists:</p><ol><li><strong>Disable</strong> shuffle-sharding on the read path</li><li><strong>Decrease</strong> the configured tenant shard size</li><li><strong>Wait</strong> at least <code>-querier.shuffle-sharding-ingesters-lookback-period</code> time</li><li><strong>Re-enable</strong> shuffle-sharding on the read path</li></ol><h3 id=query-frontend-and-query-scheduler-shuffle-sharding>Query-frontend and Query-scheduler shuffle sharding</h3><p>By default, all Cortex queriers can execute received queries for a given tenant.</p><p>When shuffle sharding is <strong>enabled</strong> by setting <code>-frontend.max-queriers-per-tenant</code> (or its respective YAML config option) to a value higher than 0 and lower than the number of available queriers, only the specified number of queriers will execute queries for a single tenant.</p><p>Note that this distribution happens in the query-frontend, or query-scheduler if used. When using query-scheduler, the <code>-frontend.max-queriers-per-tenant</code> option must be set for the query-scheduler component. When not using query-frontend (with or without scheduler), this option is not available.</p><p><em>The maximum number of queriers can be overridden on a per-tenant basis in the limits overrides configuration.</em></p><h4 id=the-impact-of-query-of-death>The impact of &ldquo;query of death&rdquo;</h4><p>In the event a tenant is repeatedly sending a &ldquo;query of death&rdquo; which leads the querier to crash or get killed because of out-of-memory, the crashed querier will get disconnected from the query-frontend or query-scheduler and a new querier will be immediately assigned to the tenant&rsquo;s shard. This practically invalidates the assumption that shuffle-sharding can be used to contain the blast radius in case of a query of death.</p><p>To mitigate it, Cortex allows you to configure a delay between when a querier disconnects because of a crash and when the crashed querier is actually removed from the tenant&rsquo;s shard (and another healthy querier is added as a replacement). A delay of 1 minute may be a reasonable trade-off:</p><ul><li>Query-frontend: <code>-query-frontend.querier-forget-delay=1m</code></li><li>Query-scheduler: <code>-query-scheduler.querier-forget-delay=1m</code></li></ul><h3 id=store-gateway-shuffle-sharding>Store-gateway shuffle sharding</h3><p>The Cortex store-gateway &ndash; used by the <a href=/docs/blocks-storage/>blocks storage</a> &ndash; by default spreads each tenant&rsquo;s blocks across all running store-gateways.</p><p>When shuffle sharding is <strong>enabled</strong> via <code>-store-gateway.sharding-strategy=shuffle-sharding</code> (or its respective YAML config option), each tenant’s blocks will be sharded across a subset of <code>-store-gateway.tenant-shard-size</code> store-gateway instances. This configuration needs to be set to <strong>store-gateway</strong>, <strong>querier</strong>, and <strong>ruler</strong>.</p><p><em>The shard size can be overridden on a per-tenant basis setting <code>store_gateway_tenant_shard_size</code> in the limits overrides configuration.</em></p><p><em>Please check out the <a href=/docs/blocks-storage/store-gateway/>store-gateway documentation</a> for more information about how it works.</em></p><h3 id=ruler-shuffle-sharding>Ruler shuffle sharding</h3><p>Cortex ruler can run in three modes:</p><ol><li><strong>No sharding at all.</strong> This is the most basic mode of the ruler. It is activated by using <code>-ruler.enable-sharding=false</code> (default) and works correctly only if a single ruler is running. In this mode, the Ruler loads all rules for all tenants.</li><li><strong>Default sharding</strong>, activated by using <code>-ruler.enable-sharding=true</code> and <code>-ruler.sharding-strategy=default</code> (default). In this mode, rulers register themselves into the ring. Each ruler will then select and evaluate only those rules that it &ldquo;owns&rdquo;.</li><li><strong>Shuffle sharding</strong>, activated by using <code>-ruler.enable-sharding=true</code> and <code>-ruler.sharding-strategy=shuffle-sharding</code>. Similarly to default sharding, rulers use the ring to distribute workload, but rule groups for each tenant can only be evaluated on a limited number of rulers (<code>-ruler.tenant-shard-size</code>, can also be set per tenant as <code>ruler_tenant_shard_size</code> in overrides).</li></ol><p>Note that when using sharding strategy, each rule group is evaluated by a single ruler only; there is no replication.</p><h3 id=compactor-shuffle-sharding>Compactor shuffle sharding</h3><p>Cortex compactor can run in three modes:</p><ol><li><strong>No sharding at all.</strong> This is the most basic mode of the compactor. It is activated by using <code>-compactor.sharding-enabled=false</code> (default). In this mode, every compactor will run every compaction.</li><li><strong>Default sharding</strong>, activated by using <code>-compactor.sharding-enabled=true</code> and <code>-compactor.sharding-strategy=default</code> (default). In this mode, compactors register themselves into the ring. One single tenant will belong to only 1 compactor.</li><li><strong>Shuffle sharding</strong>, activated by using <code>-compactor.sharding-enabled=true</code> and <code>-compactor.sharding-strategy=shuffle-sharding</code>. Similarly to default sharding, but compactions for each tenant can be carried out on multiple compactors (<code>-compactor.tenant-shard-size</code>, can also be set per tenant as <code>compactor_tenant_shard_size</code> in overrides).</li></ol><p>With shuffle sharding selected as the sharding strategy, a subset of the compactors will be used to handle a user based on the shard size.</p><p>The idea behind using the shuffle sharding strategy for the compactor is to further enable horizontal scalability and build tolerance for compactions that may take longer than the compaction interval.</p><h2 id=faq>FAQ</h2><h3 id=does-shuffle-sharding-add-additional-overhead-to-the-kv-store>Does shuffle sharding add additional overhead to the KV store?</h3><p>No, shuffle sharding subrings are computed client-side and are not stored in the ring. KV store sizing still depends primarily on the number of replicas (of any component that uses the ring, e.g. ingesters) and tokens per replica.</p><p>However, each tenant&rsquo;s subring is cached in memory on the client-side, which may slightly increase the memory footprint of certain components (mostly the distributor).</p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/guides/tls/index.html b/docs/guides/tls/index.html
index 249987fdfa..62255e52c8 100644
--- a/docs/guides/tls/index.html
+++ b/docs/guides/tls/index.html
@@ -5,7 +5,7 @@
 Generation of certs to configure TLS The first step to securing inter-service communication in Cortex with TLS is generating certificates. A Certifying Authority (CA) will be used for this purpose, which should be private to the organization, as any certificates signed by this CA will have permissions to communicate with the cluster."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span class=td-sidebar-nav-active-item>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/tls.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Securing%20communication%20between%20Cortex%20components%20with%20TLS" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#generation-of-certs-to-configure-tls>Generation of certs to configure TLS</a></li><li><a href=#load-certs-into-the-httpgrpc-serverclient>Load certs into the HTTP/GRPC server/client</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/tls/ aria-disabled=true class="btn-link disabled">Securing communication between Cortex components with TLS</a></li></ol></nav><div class=td-content><h1>Securing communication between Cortex components with TLS</h1><header class=article-meta></header><p>Cortex is a distributed system with significant traffic between its services.
diff --git a/docs/guides/tracing/index.html b/docs/guides/tracing/index.html
index 5b5d73e41e..d83bd2d6f1 100644
--- a/docs/guides/tracing/index.html
+++ b/docs/guides/tracing/index.html
@@ -9,7 +9,7 @@
 Configuration In order to configure Cortex to send traces, you must do two things:"><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span class=td-sidebar-nav-active-item>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/tracing.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Tracing" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#jaeger>Jaeger</a><ul><li><a href=#dependencies>Dependencies</a></li><li><a href=#configuration>Configuration</a></li></ul></li><li><a href=#opentelemetry>OpenTelemetry</a><ul><li><a href=#dependencies-1>Dependencies</a></li><li><a href=#configuration-1>Configuration</a></li><li><a href=#current-state>Current State</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/tracing/ aria-disabled=true class="btn-link disabled">Tracing</a></li></ol></nav><div class=td-content><h1>Tracing</h1><header class=article-meta></header><p>Cortex uses <a href=https://www.jaegertracing.io/>Jaeger</a> or <a href=https://opentelemetry.io/>OpenTelemetry</a> to implement distributed
diff --git a/docs/guides/zone-aware-replication/index.html b/docs/guides/zone-aware-replication/index.html
index 1efdf84d29..50d250c142 100644
--- a/docs/guides/zone-aware-replication/index.html
+++ b/docs/guides/zone-aware-replication/index.html
@@ -4,7 +4,7 @@
 It is completely possible that all the replicas for the given data are held within the same availability zone, even if the Cortex cluster spans multiple zones. Storing multiple replicas for a given data within the same availability zone poses a risk for data loss if there is an outage affecting various nodes within a zone or a full zone outage."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span class=td-sidebar-nav-active-item>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/guides/zone-replication.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Zone%20Aware%20Replication" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#distributors--ingesters-time-series-replication>Distributors / Ingesters: time-series replication</a></li><li><a href=#store-gateways-blocks-replication>Store-gateways: blocks replication</a></li><li><a href=#minimum-number-of-zones>Minimum number of zones</a></li><li><a href=#impact-on-unbalanced-zones>Impact on unbalanced zones</a></li><li><a href=#impact-on-costs>Impact on costs</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/guides/>Guides</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/guides/zone-aware-replication/ aria-disabled=true class="btn-link disabled">Zone Aware Replication</a></li></ol></nav><div class=td-content><h1>Zone Aware Replication</h1><header class=article-meta></header><p>Cortex supports data replication for different services. By default, data is transparently replicated across the whole pool of service instances, regardless of whether these instances are all running within the same availability zone (or data center, or rack) or in different ones.</p><p>It is completely possible that all the replicas for the given data are held within the same availability zone, even if the Cortex cluster spans multiple zones. Storing multiple replicas for a given data within the same availability zone poses a risk for data loss if there is an outage affecting various nodes within a zone or a full zone outage.</p><p>For this reason, Cortex optionally supports zone-aware replication. When zone-aware replication is <strong>enabled</strong>, replicas for the given data are guaranteed to span across different availability zones. This requires the Cortex cluster to run at least in a number of zones equal to the configured replication factor.</p><p>Reads from a zone-aware replication enabled Cortex Cluster can withstand zone failures as long as there are no more than <code>floor(replication factor / 2)</code> zones with failing instances.</p><p>The Cortex services supporting <strong>zone-aware replication</strong> are:</p><ul><li><strong><a href=#distributors-and-ingesters-time-series-replication>Distributors and Ingesters</a></strong></li><li><strong><a href=#store-gateways-blocks-replication>Store-gateways</a></strong> (<a href=/docs/blocks-storage/>blocks storage</a> only)</li></ul><h2 id=distributors--ingesters-time-series-replication>Distributors / Ingesters: time-series replication</h2><p>The Cortex time-series replication is used to hold multiple (typically 3) replicas of each time series in the <strong>ingesters</strong>.</p><p><strong>To enable</strong> the zone-aware replication for the ingesters you should:</p><ol><li>Configure the availability zone for each ingester via the <code>-ingester.availability-zone</code> CLI flag (or its respective YAML config option)</li><li>Rollout ingesters to apply the configured zone</li><li>Enable time-series zone-aware replication via the <code>-distributor.zone-awareness-enabled</code> CLI flag (or its respective YAML config option). Please be aware that this configuration option should be set to distributors, queriers, and rulers.</li></ol><p>The <code>-distributor.shard-by-all-labels</code> setting has an impact on read availability. When enabled, a metric is sharded across all ingesters, and the querier needs to fetch series from all ingesters. While, when disabled, a metric is sharded only across <code>&lt;replication factor></code> ingesters.</p><p>In the event of a large outage impacting ingesters in more than 1 zone, when <code>-distributor.shard-by-all-labels=true</code>, all queries will fail, while when disabled, some queries may still succeed if the ingesters holding the required metric are not impacted by the outage. To learn more about this flag, please refer to <a href=/docs/configuration/arguments/#distributor>distributor arguments</a>.</p><h2 id=store-gateways-blocks-replication>Store-gateways: blocks replication</h2><p>The Cortex <a href=/docs/blocks-storage/store-gateway/>store-gateway</a> (used only when Cortex is running with the <a href=/docs/blocks-storage/>blocks storage</a>) supports blocks sharding, used to horizontally scale blocks in a large cluster without hitting any vertical scalability limit.</p><p>To enable the zone-aware replication for the store-gateways, please refer to the <a href=/docs/blocks-storage/store-gateway/#zone-awareness>store-gateway</a> documentation.</p><h2 id=minimum-number-of-zones>Minimum number of zones</h2><p>For Cortex to function correctly, there must be at least the same number of availability zones as the replication factor. For example, if the replication factor is configured to 3 (default for time-series replication), the Cortex cluster should be spread at least over 3 availability zones.</p><p>It is safe to have more zones than the replication factor, but it cannot be less. Having fewer availability zones than the replication factor causes a replica write to be missed, and in some cases, the write fails if the availability zones count is too low.</p><h2 id=impact-on-unbalanced-zones>Impact on unbalanced zones</h2><p><strong>Cortex requires that each zone runs the same number of instances</strong> of a given service for which the zone-aware replication is enabled. This guarantees a fair split of the workload across zones.</p><p>On the contrary, if zones are unbalanced, the zones with a lower number of instances would have a higher pressure on resources utilization (eg. CPU and memory) compared to zones with a higher number of instances.</p><h2 id=impact-on-costs>Impact on costs</h2><p>Depending on the underlying infrastructure being used, deploying Cortex across multiple availability zones may cause an increase in running costs as most cloud providers charge for inter-availability zone networking. The most significant change would be for a Cortex cluster currently running in a single zone.</p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/index.html b/docs/index.html
index b0b11728b8..3cea248306 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -9,7 +9,7 @@
 Horizontally scalable: …"><meta property="og:title" content="Documentation"><meta property="og:description" content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta property="og:type" content="website"><meta property="og:url" content="/docs/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Documentation"><meta itemprop=description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Documentation"><meta name=twitter:description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span class=td-sidebar-nav-active-item>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/_index.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Documentation" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#features>Features</a></li><li><a href=#getting-started>Getting Started</a></li><li><a href=#documentation>Documentation</a></li><li><a href=#community-and-support>Community and Support</a></li><li><a href=#engage-with-our-community>Engage with Our Community</a></li><li><a href=#further-reading>Further reading</a><ul><li><a href=#talks>Talks</a></li><li><a href=#blog-posts>Blog Posts</a></li></ul></li><li><a href=#hosted-cortex>Hosted Cortex</a><ul><li><a href=#amazon-managed-service-for-prometheus-amp>Amazon Managed Service for Prometheus (AMP)</a></li></ul></li><li><a href=#emeritus-maintainers>Emeritus Maintainers</a></li><li><a href=#history-of-cortex>History of Cortex</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class="td-breadcrumbs td-breadcrumbs__single"><ol class=breadcrumb><li class="breadcrumb-item active" aria-current=page><a href=/docs/ aria-disabled=true class="btn-link disabled">Documentation</a></li></ol></nav><div class=td-content><h1>Documentation</h1><header class=article-meta></header><p><a href="https://scorecard.dev/viewer/?uri=github.com/cortexproject/cortex"><img src=https://api.securityscorecards.dev/projects/github.com/cortexproject/cortex/badge alt="OpenSSF Scorecard"></a>
diff --git a/docs/index.xml b/docs/index.xml
index 34e0f48012..8d8456f9dc 100644
--- a/docs/index.xml
+++ b/docs/index.xml
@@ -2109,6 +2109,14 @@ References to undefined variables are replaced by empty strings unless you speci
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[lazy_expanded_postings_enabled&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;boolean&amp;gt; | default = false]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Mark posting group as lazy if it fetches more keys than R * max series the&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># query should fetch. With R set to 100, a posting group which fetches 100K&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># keys will be marked as lazy if the current query only fetches 1000 series.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># This config is only valid if lazy expanded posting is enabled. 0 disables&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># the limit.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.lazy-expanded-posting-group-max-key-series-ratio&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[lazy_expanded_posting_group_max_key_series_ratio&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;float&amp;gt; | default = 100]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Controls how many series to fetch per batch in Store Gateway. Default value&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># is 10000.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.series-batch-size&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
@@ -2827,6 +2835,13 @@ References to undefined variables are replaced by empty strings unless you speci
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -distributor.ring.instance-interface-names&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[instance_interface_names&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;list of string&amp;gt; | default = [eth0 en0]]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># EXPERIMENTAL: Number of go routines to handle push calls from distributors to&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># ingesters. When no workers are available, a new goroutine will be spawned&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># automatically. If set to 0 (default), workers are disabled, and a new&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># goroutine will be created for each push request.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -distributor.num-push-workers&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#204a87;font-weight:bold">[num_push_workers&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;int&amp;gt; | default = 0]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#204a87;font-weight:bold">instance_limits&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Max ingestion rate (samples/sec) that this distributor will accept. This&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># limit is per-distributor, not per-tenant. Additional push requests will be&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
@@ -3346,17 +3361,17 @@ References to undefined variables are replaced by empty strings unless you speci
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">healthcheck_config&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The number of consecutive failed health checks required before considering&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># a target unhealthy. 0 means disabled.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -ingester.client.unhealthy-threshold&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -ingester.client.healthcheck.unhealthy-threshold&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[unhealthy_threshold&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;int&amp;gt; | default = 0]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The approximate amount of time between health checks of an individual&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># target.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -ingester.client.interval&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -ingester.client.healthcheck.interval&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[interval&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;duration&amp;gt; | default = 5s]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The amount of time during which no response from a target means a failed&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># health check.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -ingester.client.timeout&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -ingester.client.healthcheck.timeout&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[timeout&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;duration&amp;gt; | default = 1s]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Max inflight push requests that this ingester client can handle. This limit is&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
@@ -4069,17 +4084,17 @@ References to undefined variables are replaced by empty strings unless you speci
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">healthcheck_config&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The number of consecutive failed health checks required before considering&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># a target unhealthy. 0 means disabled.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.unhealthy-threshold&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.healthcheck.unhealthy-threshold&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[unhealthy_threshold&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;int&amp;gt; | default = 0]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The approximate amount of time between health checks of an individual&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># target.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.interval&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.healthcheck.interval&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[interval&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;duration&amp;gt; | default = 5s]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The amount of time during which no response from a target means a failed&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># health check.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.timeout&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.healthcheck.timeout&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[timeout&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;duration&amp;gt; | default = 1s]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">&lt;/span>&lt;span style="color:#8f5902;font-style:italic"># If enabled, store gateway query stats will be logged using `info` log level.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
@@ -8872,17 +8887,17 @@ The domain name after the prefix is looked up as a SRV query, with no A/AAAA loo
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">healthcheck_config&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The number of consecutive failed health checks required before&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># considering a target unhealthy. 0 means disabled.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.unhealthy-threshold&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.healthcheck.unhealthy-threshold&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[unhealthy_threshold&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;int&amp;gt; | default = 0]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The approximate amount of time between health checks of an individual&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># target.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.interval&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.healthcheck.interval&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[interval&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;duration&amp;gt; | default = 5s]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># The amount of time during which no response from a target means a failed&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># health check.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
-&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.timeout&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -querier.store-gateway-client.healthcheck.timeout&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[timeout&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;duration&amp;gt; | default = 1s]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># If enabled, store gateway query stats will be logged using `info` log level.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
@@ -10073,6 +10088,14 @@ The domain name after the prefix is looked up as a SRV query, with no A/AAAA loo
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[lazy_expanded_postings_enabled&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;boolean&amp;gt; | default = false]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Mark posting group as lazy if it fetches more keys than R * max series the&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># query should fetch. With R set to 100, a posting group which fetches 100K&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># keys will be marked as lazy if the current query only fetches 1000 series.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># This config is only valid if lazy expanded posting is enabled. 0 disables&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># the limit.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.lazy-expanded-posting-group-max-key-series-ratio&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[lazy_expanded_posting_group_max_key_series_ratio&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;float&amp;gt; | default = 100]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Controls how many series to fetch per batch in Store Gateway. Default&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># value is 10000.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.series-batch-size&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
@@ -12092,6 +12115,14 @@ Displays the status of the store-gateways ring, including the tokens owned by ea
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[lazy_expanded_postings_enabled&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;boolean&amp;gt; | default = false]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Mark posting group as lazy if it fetches more keys than R * max series the&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># query should fetch. With R set to 100, a posting group which fetches 100K&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># keys will be marked as lazy if the current query only fetches 1000 series.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># This config is only valid if lazy expanded posting is enabled. 0 disables&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># the limit.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.lazy-expanded-posting-group-max-key-series-ratio&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#204a87;font-weight:bold">[lazy_expanded_posting_group_max_key_series_ratio&lt;/span>&lt;span style="color:#000;font-weight:bold">:&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#000">&amp;lt;float&amp;gt; | default = 100]&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
+&lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># Controls how many series to fetch per batch in Store Gateway. Default&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># value is 10000.&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
 &lt;/span>&lt;/span>&lt;/span>&lt;span style="display:flex;">&lt;span>&lt;span style="color:#f8f8f8;text-decoration:underline"> &lt;/span>&lt;span style="color:#8f5902;font-style:italic"># CLI flag: -blocks-storage.bucket-store.series-batch-size&lt;/span>&lt;span style="color:#f8f8f8;text-decoration:underline">
diff --git a/docs/operations/index.html b/docs/operations/index.html
index aa483a6199..778ca16c50 100644
--- a/docs/operations/index.html
+++ b/docs/operations/index.html
@@ -1,7 +1,7 @@
 <!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.101.0"><link rel=alternate type=application/rss+xml href=/docs/operations/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/favicons/android-96x96.png sizes=96x96><link rel=icon type=image/png href=/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/favicons/android-192x192.png sizes=192x192><title>Operating Cortex | Cortex</title><meta name=description content><meta property="og:title" content="Operating Cortex"><meta property="og:description" content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta property="og:type" content="website"><meta property="og:url" content="/docs/operations/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Operating Cortex"><meta itemprop=description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Operating Cortex"><meta name=twitter:description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 active td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span class=td-sidebar-nav-active-item>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/operations/_index.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Operating%20Cortex" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/operations/ aria-disabled=true class="btn-link disabled">Operations</a></li></ol></nav><div class=td-content><h1>Operating Cortex</h1><header class=article-meta></header><ul><li><a href=/docs/operations/query-auditor/>Query Auditor (tool)</a></li><li><a href=/docs/operations/query-tee/>Query Tee (service)</a></li><li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/>Requests mirroring to secondary cluster</a></li><li><a href=/docs/operations/scaling-query-frontend/>Scaling the Query Frontend</a></li></ul></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/operations/query-auditor/index.html b/docs/operations/query-auditor/index.html
index e87067c7b8..c224cceec2 100644
--- a/docs/operations/query-auditor/index.html
+++ b/docs/operations/query-auditor/index.html
@@ -4,7 +4,7 @@
 How it works The query-audit tool performs a set of queries against two backends that expose the Prometheus read API. This is generally the query-frontend component of two Cortex deployments."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span class=td-sidebar-nav-active-item>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/operations/query-auditor.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Query%20Auditor%20%28tool%29" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#how-it-works>How it works</a><ul><li><a href=#limitations>Limitations</a></li><li><a href=#use-cases>Use cases</a></li><li><a href=#example-configuration>Example Configuration</a></li><li><a href=#example-output>Example Output</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/operations/>Operations</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/operations/query-auditor/ aria-disabled=true class="btn-link disabled">Query Auditor (tool)</a></li></ol></nav><div class=td-content><h1>Query Auditor (tool)</h1><header class=article-meta></header><p>The query auditor is a tool bundled in the Cortex repository, but <strong>not</strong> included in Docker images &ndash; this must be built from source. It&rsquo;s primarily useful for those <em>developing</em> Cortex, but can be helpful to operators as well during certain scenarios (backend migrations come to mind).</p><h2 id=how-it-works>How it works</h2><p>The <code>query-audit</code> tool performs a set of queries against two backends that expose the Prometheus read API. This is generally the <code>query-frontend</code> component of two Cortex deployments. It will then compare the differences in the responses to determine the average difference for each query. It does this by:</p><ul><li>Ensuring the resulting label sets match.</li><li>For each label set, ensuring they contain the same number of samples as their pair from the other backend.</li><li>For each sample, calculates their difference against it&rsquo;s pair from the other backend/label set.</li><li>Calculates the average diff per query from the above diffs.</li></ul><h3 id=limitations>Limitations</h3><p>It currently only supports queries with <code>Matrix</code> response types.</p><h3 id=use-cases>Use cases</h3><ul><li>Correctness testing when working on the read path.</li><li>Comparing results from different backends.</li></ul><h3 id=example-configuration>Example Configuration</h3><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-yaml data-lang=yaml><span style=display:flex><span><span style=color:#204a87;font-weight:700>control</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline>
diff --git a/docs/operations/query-tee/index.html b/docs/operations/query-tee/index.html
index c0e25ae8c8..9b3316b155 100644
--- a/docs/operations/query-tee/index.html
+++ b/docs/operations/query-tee/index.html
@@ -7,7 +7,7 @@
 How to run it You can run query-tee in two ways:"><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span class=td-sidebar-nav-active-item>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/operations/query-tee.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Query%20Tee%20%28service%29" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#how-to-run-it>How to run it</a></li><li><a href=#how-it-works>How it works</a><ul><li><a href=#api-endpoints>API endpoints</a></li><li><a href=#authentication>Authentication</a></li><li><a href=#backend-response-selection>Backend response selection</a></li><li><a href=#backend-results-comparison>Backend results comparison</a></li><li><a href=#slow-backends>Slow backends</a></li><li><a href=#exported-metrics>Exported metrics</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/operations/>Operations</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/operations/query-tee/ aria-disabled=true class="btn-link disabled">Query Tee (service)</a></li></ol></nav><div class=td-content><h1>Query Tee (service)</h1><header class=article-meta></header><p>The <code>query-tee</code> is a standalone service which can be used for testing purposes to compare the query performances of 2+ backend systems (ie. Cortex clusters) ingesting the same exact series.</p><p>This service exposes Prometheus-compatible read API endpoints and, for each received request, performs the request against all backends tracking the response time of each backend and then sends back to the client one of the received responses.</p><h2 id=how-to-run-it>How to run it</h2><p>You can run <code>query-tee</code> in two ways:</p><ul><li>Build it from sources<pre tabindex=0><code>go run ./cmd/query-tee -help
diff --git a/docs/operations/requests-mirroring-to-secondary-cluster/index.html b/docs/operations/requests-mirroring-to-secondary-cluster/index.html
index a65eacff59..31bc90baf1 100644
--- a/docs/operations/requests-mirroring-to-secondary-cluster/index.html
+++ b/docs/operations/requests-mirroring-to-secondary-cluster/index.html
@@ -5,7 +5,7 @@
 For example, requests mirroring can be used when you need to setup a testing Cortex cluster receiving the same series ingested by a primary one without having control over Prometheus remote write config (if you do, then configuring two remote write entries in Prometheus would be the preferred option)."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span class=td-sidebar-nav-active-item>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/operations/requests-mirroring-to-secondary-cluster.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Requests%20mirroring%20to%20secondary%20cluster" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#mirroring-with-envoy-proxy>Mirroring with Envoy proxy</a><ul><li><a href=#example-envoy-config>Example Envoy config</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/operations/>Operations</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ aria-disabled=true class="btn-link disabled">Requests mirroring to secondary cluster</a></li></ol></nav><div class=td-content><h1>Requests mirroring to secondary cluster</h1><header class=article-meta></header><p>Requests mirroring (or shadowing) is a technique you can use to mirror requests from a primary Cortex cluster to a secondary one.</p><p>For example, requests mirroring can be used when you need to setup a testing Cortex cluster receiving the same series ingested by a primary one without having control over Prometheus remote write config (if you do, then configuring two remote write entries in Prometheus would be the preferred option).</p><h2 id=mirroring-with-envoy-proxy>Mirroring with Envoy proxy</h2><p><a href=https://www.envoyproxy.io/>Envoy proxy</a> can be used to mirror HTTP requests to a secondary upstream cluster. From a network path perspective, you should run Envoy in front of both clusters distributors, letting Envoy to proxy requests to the primary Cortex cluster and mirror them to a secondary cluster in background. The performances and availability of the secondary cluster have no impact on the requests to the primary one. The response to the client will always be the one from the primary one. In this sense, the requests from Envoy to the secondary cluster are &ldquo;fire and forget&rdquo;.</p><h3 id=example-envoy-config>Example Envoy config</h3><p>The following Envoy configuration shows an example with two Cortex clusters. Envoy will listen on port <code>9900</code> and will proxies all requests to <code>cortex-primary:80</code>, mirroring it to <code>cortex-secondary:80</code> too.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-yaml data-lang=yaml><span style=display:flex><span><span style=color:#204a87;font-weight:700>admin</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline>
diff --git a/docs/operations/scaling-query-frontend/index.html b/docs/operations/scaling-query-frontend/index.html
index 34449d467e..164d8f933a 100644
--- a/docs/operations/scaling-query-frontend/index.html
+++ b/docs/operations/scaling-query-frontend/index.html
@@ -5,7 +5,7 @@
 Scaling the Query Frontend For every query frontend the querier adds a configurable number of concurrent workers which are each capable of executing a query. Each worker is connected to a single query frontend instance, therefore scaling up the query frontend impacts the amount of work each individual querier is attempting to do at any given time."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span class=td-sidebar-nav-active-item>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/operations/scalable-query-frontend.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Scaling%20the%20Query%20Frontend" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#scaling-the-query-frontend>Scaling the Query Frontend</a><ul><li><a href=#querier-max-concurrency>Querier Max Concurrency</a></li><li><a href=#query-scheduler>Query Scheduler</a></li><li><a href=#dns-configuration--readiness>DNS Configuration / Readiness</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/operations/>Operations</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/operations/scaling-query-frontend/ aria-disabled=true class="btn-link disabled">Scaling the Query Frontend</a></li></ol></nav><div class=td-content><h1>Scaling the Query Frontend</h1><header class=article-meta></header><p>Historically scaling the Cortex query frontend has <a href=/docs/proposals/scalable-query-frontend/>posed some challenges</a>.
diff --git a/docs/proposals/auth-gateway/index.html b/docs/proposals/auth-gateway/index.html
index 64ea401f23..7978aafd21 100644
--- a/docs/proposals/auth-gateway/index.html
+++ b/docs/proposals/auth-gateway/index.html
@@ -7,7 +7,7 @@
 If you run Cortex for multiple tenants you need to identify your tenants every time …"><meta property="og:title" content="Authentication Gateway"><meta property="og:description" content="Author: Doğukan Teber Date: March 2023 Status: Proposed Overview If you run Cortex for multiple tenants you need to identify your tenants every time they send metrics or query them. This is needed to ensure that metrics can be ingested and queried separately from each other. For this purpose, the Cortex microservices require you to pass a Header called X-Scope-OrgID. The Cortex k8s manifests suggest deploying an NGINX server inside of each tenant which acts as a reverse proxy."><meta property="og:type" content="article"><meta property="og:url" content="/docs/proposals/auth-gateway/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Authentication Gateway"><meta itemprop=description content="Author: Doğukan Teber Date: March 2023 Status: Proposed Overview If you run Cortex for multiple tenants you need to identify your tenants every time they send metrics or query them. This is needed to ensure that metrics can be ingested and queried separately from each other. For this purpose, the Cortex microservices require you to pass a Header called X-Scope-OrgID. The Cortex k8s manifests suggest deploying an NGINX server inside of each tenant which acts as a reverse proxy."><meta itemprop=wordCount content="410"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Authentication Gateway"><meta name=twitter:description content="Author: Doğukan Teber Date: March 2023 Status: Proposed Overview If you run Cortex for multiple tenants you need to identify your tenants every time they send metrics or query them. This is needed to ensure that metrics can be ingested and queried separately from each other. For this purpose, the Cortex microservices require you to pass a Header called X-Scope-OrgID. The Cortex k8s manifests suggest deploying an NGINX server inside of each tenant which acts as a reverse proxy."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span class=td-sidebar-nav-active-item>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/auth-gateway.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Authentication%20Gateway" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/auth-gateway/ aria-disabled=true class="btn-link disabled">Authentication Gateway</a></li></ol></nav><div class=td-content><h1>Authentication Gateway</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/dogukanteber>Doğukan Teber</a></li><li>Date: March 2023</li><li>Status: Proposed</li></ul><h2 id=overview>Overview</h2><p>If you run Cortex for multiple tenants you need to identify your tenants every time they send metrics or query them.
diff --git a/docs/proposals/block-storage-time-series-deletion/index.html b/docs/proposals/block-storage-time-series-deletion/index.html
index d2730c3cda..a1f264c312 100644
--- a/docs/proposals/block-storage-time-series-deletion/index.html
+++ b/docs/proposals/block-storage-time-series-deletion/index.html
@@ -10,7 +10,7 @@
 This can be very important for users to have as confidential or accidental data might have been incorrectly pushed and needs to be removed."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span class=td-sidebar-nav-active-item>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/block-storage-time-series-deletion.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Time%20Series%20Deletion%20from%20Blocks%20Storage" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#problem>Problem</a></li><li><a href=#related-works>Related works</a></li><li><a href=#background-on-current-storage>Background on current storage</a></li><li><a href=#proposal>Proposal</a><ul><li><a href=#api-endpoints>API Endpoints</a></li><li><a href=#deletion-lifecycle>Deletion Lifecycle</a></li><li><a href=#filtering-data-during-queries-while-not-yet-deleted>Filtering data during queries while not yet deleted:</a></li><li><a href=#permanently-deleting-the-data>Permanently deleting the data</a></li></ul></li><li><a href=#current-open-questions>Current Open Questions:</a></li><li><a href=#alternatives-considered>Alternatives Considered</a><ul><li></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/block-storage-time-series-deletion/ aria-disabled=true class="btn-link disabled">Time Series Deletion from Blocks Storage</a></li></ol></nav><div class=td-content><h1>Time Series Deletion from Blocks Storage</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/ilangofman>Ilan Gofman</a></li><li>Date: June 2021</li><li>Status: Proposal</li></ul><h2 id=problem>Problem</h2><p>Currently, Cortex only implements a time series deletion API for chunk storage. We present a design for implementing time series deletion with block storage. We would like to have the same API for deleting series as currently implemented in Prometheus and in Cortex with chunk storage.</p><p>This can be very important for users to have as confidential or accidental data might have been incorrectly pushed and needs to be removed. As well as potentially removing high cardinality data that is causing inefficient queries.</p><h2 id=related-works>Related works</h2><p>As previously mentioned, the deletion feature is already implemented with chunk storage. The main functionality is implemented through the purger service. It accepts requests for deletion and processes them. At first, when a deletion request is made, a tombstone is created. This is used to filter out the data for queries. After some time, a deletion plan is executed where the data is permanently removed from chunk storage.</p><p>Can find more info here:</p><ul><li><a href=https://cortexmetrics.io/docs/guides/deleting-series/>Cortex documentation for chunk store deletion</a></li><li><a href=https://docs.google.com/document/d/1PeKwP3aGo3xVrR-2qJdoFdzTJxT8FcAbLm2ew_6UQyQ/edit>Chunk deletion proposal</a></li></ul><h2 id=background-on-current-storage>Background on current storage</h2><p>With a block-storage configuration, Cortex stores data that could be potentially deleted by a user in:</p><ul><li>Object store (GCS, S3, etc..) for long term storage of blocks</li><li>Ingesters for more recent data that should be eventually transferred to the object store</li><li>Cache<ul><li>Index cache</li><li>Metadata cache</li><li>Chunks cache (stores the potentially to be deleted data)</li><li>Query results cache (stores the potentially to be deleted data)</li></ul></li><li>Compactor during the compaction process</li><li>Store-gateway</li></ul><h2 id=proposal>Proposal</h2><p>The deletion will not happen right away. Initially, the data will be filtered out from queries using tombstones and will be deleted afterward. This will allow the user some time to cancel the delete request.</p><h3 id=api-endpoints>API Endpoints</h3><p>The existing purger service will be used to process the incoming requests for deletion. The API will follow the same structure as the chunk storage endpoints for deletion, which is also based on the Prometheus deletion API.</p><p>This will enable the following endpoints for Cortex when using block storage:</p><p><code>POST /api/v1/admin/tsdb/delete_series</code> - Accepts <a href=https://prometheus.io/docs/prometheus/latest/querying/api/#delete-series>Prometheus style delete request</a> for deleting series.</p><p>Parameters:</p><ul><li><code>start=&lt;rfc3339 | unix_timestamp></code><ul><li>Optional. If not provided, will be set to minimum possible time.</li></ul></li><li><code>end=&lt;rfc3339 | unix_timestamp></code><ul><li>Optional. If not provided, will be set to maximum possible time (time when request was made). End time cannot be greater than the current UTC time.</li></ul></li><li><code>match[]=&lt;series_selector></code><ul><li>Cannot be empty, must contain at least one label matcher argument.</li></ul></li></ul><p><code>POST /api/v1/admin/tsdb/cancel_delete_request</code> - To cancel a request if it has not been processed yet for permanent deletion. This can only be done before the <code>-purger.delete-request-cancel-period</code> has passed.
diff --git a/docs/proposals/blocks-storage-bucket-index/index.html b/docs/proposals/blocks-storage-bucket-index/index.html
index aac2cce73b..6d80dba083 100644
--- a/docs/proposals/blocks-storage-bucket-index/index.html
+++ b/docs/proposals/blocks-storage-bucket-index/index.html
@@ -10,7 +10,7 @@
 For each discovered block, they need to fetch the meta.json file and check for the existence of deletion-mark.json (used to signal a block is &ldquo;marked for deletion&rdquo;). The meta.json file is immutable and gets cached, but deletion-mark."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span class=td-sidebar-nav-active-item>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/blocks-storage-bucket-index.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Blocks%20storage%20bucket%20index" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#introduction>Introduction</a></li><li><a href=#goal>Goal</a></li><li><a href=#out-of-scope>Out of scope</a></li><li><a href=#proposal>Proposal</a><ul><li><a href=#bucket-index-structure>Bucket index structure</a></li><li><a href=#why-the-compactor-writes-the-bucket-index>Why the compactor writes the bucket index</a></li><li><a href=#how-to-reduce-bucket-api-calls-required-to-discover-deletion-marks>How to reduce bucket API calls required to discover deletion marks</a></li><li><a href=#what-if-a-bucket-index-gets-stale>What if a bucket index gets stale</a></li><li><a href=#object-store-eventual-consistency>Object store eventual consistency</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/blocks-storage-bucket-index/ aria-disabled=true class="btn-link disabled">Blocks storage bucket index</a></li></ol></nav><div class=td-content><h1>Blocks storage bucket index</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/pracucci>Marco Pracucci</a></li><li>Date: November 2020</li><li>Status: draft</li></ul><h2 id=introduction>Introduction</h2><p>Queriers and store-gateways, at startup and periodically while running, need to scan the entire object store bucket to find tenants and blocks for each tenants.</p><p>For each discovered block, they need to fetch the <code>meta.json</code> file and check for the existence of <code>deletion-mark.json</code> (used to signal a block is &ldquo;marked for deletion&rdquo;). The <code>meta.json</code> file is immutable and gets cached, but <code>deletion-mark.json</code> can be created at any time and its non-existence can&rsquo;t be safely cached for long (actually our cache is pretty ineffective regarding this).</p><p>The number of blocks in the storage linearly increases with the number of tenants, and so does the number of bucket API calls required to scan the bucket.</p><p>For example, assuming 1 block / day / tenant and a 400d retention, we would have 400 blocks for a single-tenant cluster, but 4M blocks for a 10K tenants cluster (regardless of the number of active series or their QPS).</p><p>We&rsquo;re currently testing a cluster running with 6K tenants and we have faced the following issues:</p><ul><li>The cold start of queriers and store-gateways (empty caches) take tens of minutes.</li><li>If we increase the concurrency of bucket operations run to scan the bucket, we hit into object store rate limits and/or the 5xx error rate increases</li><li>The need to run &ldquo;list objects&rdquo; API call for queriers and store-gateways to discover blocks and the check for the existence of <code>deletion-mark.json</code> (which requires a read operation for each block and can&rsquo;t be cached for a long time) represents a significant % of bucket API calls baseline costs, regardless the tenants QPS (costs we have even if the cluster has 0 queries)</li></ul><h2 id=goal>Goal</h2><p>The goal of this proposal is:</p><ul><li>The querier should be up and running without having to scan the bucket at all (zero startup time)</li><li>The querier should not run any &ldquo;list objects&rdquo; operation at anytime</li><li>The querier should require only 1 &ldquo;get object&rdquo; operation to get the entire view of a tenant&rsquo;s blocks</li></ul><h2 id=out-of-scope>Out of scope</h2><p>We believe the same technique described in this proposal could be applied to optimise the store-gateway too (we&rsquo;ve built a <a href=https://github.com/cortexproject/cortex/compare/pracucci:experiment-object-store-based-blocks-index>PoC</a>), however to keep the design doc easier to discuss we suggest to keep the store-gateway out of scope and address it in a follow-up design doc.</p><h2 id=proposal>Proposal</h2><p>We propose to introduce a per-tenant bucket index. The index is a single JSON file containing two main information: list of all completed (non partial) blocks in the bucket + list of all deletion marks. The bucket index is stored in the bucket within the tenant location (eg. <code>/user-1/bucket-index.json</code>) and is kept updated by the compactor.</p><p>The querier, at query time, checks whether the bucket index for the tenant has already been loaded in memory. If not, the querier will download it and cache it in memory. Given it&rsquo;s a small file, we expect the lazy download of the bucket index to not significantly impact first query performances.</p><p>While in-memory, a background process will keep it updated at periodic intervals (configurable), so that subsequent queries from the same tenant to the same querier instance will use the cached (and periodically updated) bucket index.</p><p>If a bucket index is unused for a long time (configurable), eg. because that querier instance is not receiving any query from the tenant, the querier will offload it, stopping to keep it updated at regular intervals. This is particularly useful when shuffle sharding is enabled, because a querier will only run queries for a subset of tenants and tenants can be re-sharded across queries in the event of a scale up/down or rolling updates.</p><p>The bucket index will be also cached on memcached for a short period of time, to reduce the number of &ldquo;get object&rdquo; operations in case multiple queriers fetch it in a short period of time (eg. 5 minutes).</p><h3 id=bucket-index-structure>Bucket index structure</h3><p>The <code>bucket-index.json</code> is a single file containing the following information:</p><ul><li><code>Version</code></li><li>List of blocks<ul><li><code>ID</code>, <code>MinTime</code>, <code>MaxTime</code>, <code>UploadedAt</code></li><li><code>SegmentsFormat</code> (eg. &ldquo;1-based-6-digits&rdquo;), <code>SegmentsNum</code> (these two information will be used by the store-gateway)</li></ul></li><li>List of block deletion marks<ul><li><code>ID</code>, <code>DeletionTime</code></li></ul></li><li>Timestamp of when the index has been updated</li></ul><p>The bucket index intentionally stores a subset of data for each block&rsquo;s <code>meta.json</code>, in order to keep the size of the index small. The information in the index is enough to run the querier without having to load the full <code>meta.json</code> of each block.</p><p><strong>Size</strong>: the expected size of the index is about 150 bytes per block. In the case of a tenant with 400 blocks, its bucket index would be 58KB. The size could be further reduced compressing it: experiments show a compression ratio of about 4x using gzip.</p><h3 id=why-the-compactor-writes-the-bucket-index>Why the compactor writes the bucket index</h3><p>There are few reasons why the compactor may be a good candidate to write the bucket index:</p><ul><li>The compactor already scans the bucket to have a consistent view of the blocks before running a compaction. Writing the bucket index in the compactor would require no additional bucket API calls.</li><li>Queriers and store-gateways currently read only from the bucket (no writes). We believe that it may be a nice property to preserve.</li></ul><h3 id=how-to-reduce-bucket-api-calls-required-to-discover-deletion-marks>How to reduce bucket API calls required to discover deletion marks</h3><p>The <code>deletion-mark.json</code> files are stored within each block location. This means that the compactor would still need to run a &ldquo;get object&rdquo; operation for every single block in order to find out which block has it, while updating the bucket index.</p><p>To reduce the number of operations required, we propose to store block deletion marks for all blocks in a per-tenant location (<code>markers/</code>). Discovering all blocks marked for deletion for a given tenant would only require a single &ldquo;list objects&rdquo; operation on the <code>/&lt;tenant-id>/markers/</code> prefix. New markers introduced in the future (eg. tenant deletion mark) could be stored in the same location in order to discover all markers with a single &ldquo;list objects&rdquo; operation.</p><p>For example:</p><pre tabindex=0><code>/user-1/markers/01ER1ZSYHF1FT6RBD8HTVQWX13-deletion-mark.json
diff --git a/docs/proposals/blocks-storage-sharding/index.html b/docs/proposals/blocks-storage-sharding/index.html
index 1e88f8b9e0..5f1bf444e5 100644
--- a/docs/proposals/blocks-storage-sharding/index.html
+++ b/docs/proposals/blocks-storage-sharding/index.html
@@ -7,7 +7,7 @@
 In Cortex, when using the experimental blocks storage, each querier internally runs …"><meta property="og:title" content="Blocks storage sharding"><meta property="og:description" content="Author: Marco Pracucci Date: March 2020 Status: accepted Problem In Cortex, when using the experimental blocks storage, each querier internally runs the Thanos BucketStore. This means that each querier has a full view over all blocks in the long-term storage and all blocks index headers are loaded in each querier memory. The querier memory usage linearly increase with number and size of all blocks in the storage, imposing a scalability limit to the blocks storage."><meta property="og:type" content="article"><meta property="og:url" content="/docs/proposals/blocks-storage-sharding/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Blocks storage sharding"><meta itemprop=description content="Author: Marco Pracucci Date: March 2020 Status: accepted Problem In Cortex, when using the experimental blocks storage, each querier internally runs the Thanos BucketStore. This means that each querier has a full view over all blocks in the long-term storage and all blocks index headers are loaded in each querier memory. The querier memory usage linearly increase with number and size of all blocks in the storage, imposing a scalability limit to the blocks storage."><meta itemprop=wordCount content="1135"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Blocks storage sharding"><meta name=twitter:description content="Author: Marco Pracucci Date: March 2020 Status: accepted Problem In Cortex, when using the experimental blocks storage, each querier internally runs the Thanos BucketStore. This means that each querier has a full view over all blocks in the long-term storage and all blocks index headers are loaded in each querier memory. The querier memory usage linearly increase with number and size of all blocks in the storage, imposing a scalability limit to the blocks storage."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span class=td-sidebar-nav-active-item>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/blocks-storage-sharding.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Blocks%20storage%20sharding" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#problem>Problem</a></li><li><a href=#proposed-solution>Proposed solution</a><ul><li><a href=#ring-based-sharding-and-replication>Ring-based sharding and replication</a></li><li><a href=#query-execution>Query execution</a></li><li><a href=#query-results-consistency>Query results consistency</a></li></ul></li><li><a href=#trade-offs>Trade-offs</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/blocks-storage-sharding/ aria-disabled=true class="btn-link disabled">Blocks storage sharding</a></li></ol></nav><div class=td-content><h1>Blocks storage sharding</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/pracucci>Marco Pracucci</a></li><li>Date: March 2020</li><li>Status: accepted</li></ul><h2 id=problem>Problem</h2><p>In Cortex, when using the experimental blocks storage, each querier internally runs the Thanos <a href=https://github.com/thanos-io/thanos/blob/master/pkg/store/bucket.go><code>BucketStore</code></a>. This means that each querier has a full view over all blocks in the long-term storage and all blocks index headers are loaded in each querier memory. The querier memory usage linearly increase with number and size of all blocks in the storage, imposing a scalability limit to the blocks storage.</p><p>In this proposal we want to solve this. In particular, we want to:</p><ol><li>Shard blocks (index headers) across a pool of nodes</li><li>Do not compromise HA on the read path (if a node fails, queries should continue to work)</li><li>Do not compromise correctness (either the query result is correct or it fails)</li></ol><h2 id=proposed-solution>Proposed solution</h2><p>The idea is to introduce a new Cortex service - <code>store-gateway</code> - internally running the Thanos <a href=https://github.com/thanos-io/thanos/blob/master/pkg/store/bucket.go><code>BucketStore</code></a>. At query time, a querier will run a query fetching the matching series both from ingesters and the subset of gateways holding the related blocks (based on the query time range). Blocks are replicated across the gateways in order to guarantee query results consistency and HA even in the case of a gateway instance failure.</p><h3 id=ring-based-sharding-and-replication>Ring-based sharding and replication</h3><p>In order to build blocks sharding and replication, the <code>store-gateway</code> instances form a <a href=/docs/architecture/#the-hash-ring>ring</a>. Each gateway instance uses a custom <a href=https://github.com/thanos-io/thanos/blob/master/pkg/block/fetcher.go#L108><code>MetaFetcherFilter</code></a> to filter blocks loaded on the instance itself, keeping only blocks whose <code>hash(block-id)</code> is within the tokens range assigned to the gateway instance within the ring.</p><p>Within a gateway, the blocks synchronization is triggered in two cases:</p><ol><li><strong>Periodically</strong><br>to discover new blocks uploaded by ingesters or compactor, and delete old blocks removed due to retention or by the compactor</li><li><strong>On-demand</strong><br>when the ring topology changes (the tokens ranges assigned to the gateway instance have changed)</li></ol><p>It&rsquo;s important to outline that the sync takes time (typically will have to re-scan the bucket and download new blocks index headers) and Cortex needs to guarantee query results consistency at any given time (<em>see below</em>).</p><h3 id=query-execution>Query execution</h3><p>When a querier executes a query, it will need to fetch series both from ingesters and the store-gateway instances.</p><p>For a given query, the number of blocks to query is expected to be low, especially if the Cortex cluster is running the <code>query-frontend</code> with a <code>24h</code> query split interval. In this scenario, whatever is the client&rsquo;s query time range, the <code>query-frontend</code> will split the client&rsquo;s query into partitioned queries each with up to <code>24h</code> time range and the querier will likely hit not more than 1 block per partitioned query (except for the last 24h for which blocks may have not been compacted yet).</p><p>Given this assumption, we want to avoid sending every query to every store-gateway instance. The querier should be able to take an informed decision about the minimum subset of store-gateway instances which needs to query given a time range.</p><p>The idea is to run the <a href=https://github.com/thanos-io/thanos/blob/master/pkg/block/fetcher.go#L127><code>MetaFetcher</code></a> also within the querier, but without any sharding filter (contrary to the store-gateway). At any given point in time, the querier knows the entire list of blocks in the storage. When the querier executes the <code>Select()</code> (or <code>SelectSorted()</code>) it does:</p><ol><li>Compute the list of blocks by the query time range</li><li>Compute the minimum list of store-gateway instances containing the required blocks (using the information from the ring)</li><li>Fetch series from ingesters and the matching store-gateway instances</li><li>Merge and deduplicate received series<ul><li>Optimization: can be skipped if the querier hits only 1 store-gateway</li></ul></li></ol><h3 id=query-results-consistency>Query results consistency</h3><p>When a querier executes a query, it should guarantee that either all blocks matching the time range are queried or the query fails.</p><p>However, due to the (intentional) lack of a strong coordination between queriers and store-gateways, and the ring topology which can change any time, there&rsquo;s no guarantee that the blocks assigned to a store-gateway shard are effectively loaded on the store-gateway itself at any given point in time.</p><p>The idea is introduce a <strong>consistency check in the querier</strong>. When a store-gateway receives a request from the querier, the store-gateway includes in the response the list of block IDs currently loaded on the store-gateway itself. The querier can then merge the list of block IDs received from all store-gateway hit, and match it against the list of block IDs computed at the beginning of the query execution.</p><p>There are three possible scenarios:</p><ol><li>The list match: all good</li><li>All the blocks known by the querier are within the list of blocks returned by store-gateway, but the store-gateway also included blocks unknown to the querier: all good (it means the store-gateways have discovered and loaded new blocks before the querier discovered them)</li><li>Some blocks known by the querier are <strong>not</strong> within the list of blocks returned by store-gateway: potential consistency issue</li></ol><p>We want to protect from a partial results response which may occur in the case #3. However, there are some legit cases which, if not handled, would lead to frequent false positives. Given the querier and store-gateway instances independently scan the bucket at a regular interval (to find new blocks or deleted blocks), we may be in one of the following cases:</p><p>a. The querier has discovered new blocks before the store-gateway successfully discovered and loaded them
diff --git a/docs/proposals/cross-tenant-query-federation/index.html b/docs/proposals/cross-tenant-query-federation/index.html
index 2c83f8c216..24f15b0f6c 100644
--- a/docs/proposals/cross-tenant-query-federation/index.html
+++ b/docs/proposals/cross-tenant-query-federation/index.html
@@ -13,7 +13,7 @@
 Alternatives considered Aggregation in PromQL API clients In theory PromQL API clients could be aggregating/correlating query results from multiple tenants."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span class=td-sidebar-nav-active-item>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/cross-tenant-query-federation.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Cross-Tenant%20Query%20Federation" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#overview>Overview</a></li><li><a href=#reasoning>Reasoning</a></li><li><a href=#alternatives-considered>Alternatives considered</a><ul><li><a href=#aggregation-in-promql-api-clients>Aggregation in PromQL API clients</a></li><li><a href=#multi-tenant-aggregation-in-the-query-frontends>Multi tenant aggregation in the query frontends</a></li></ul></li><li><a href=#challenges>Challenges</a><ul><li><a href=#aggregate-data-without-overlaps>Aggregate data without overlaps</a></li><li><a href=#exposure-of-feature-to-the-user>Exposure of feature to the user</a></li><li><a href=#implementing-limits-fairness-and-observability-for-cross-tenant-queries>Implementing Limits, Fairness and Observability for Cross-Tenant queries</a></li></ul></li><li><a href=#conclusion>Conclusion</a><ul><li><a href=#future-work>Future work</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/cross-tenant-query-federation/ aria-disabled=true class="btn-link disabled">Cross-Tenant Query Federation</a></li></ol></nav><div class=td-content><h1>Cross-Tenant Query Federation</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/simonswine>Christian Simon</a></li><li>Date: October 2020</li><li>Status: Accepted</li></ul><h2 id=overview>Overview</h2><p>This document aims to describe how to implement the ability to allow queries to cover data from more than a single Cortex tenant.</p><h2 id=reasoning>Reasoning</h2><p>Adopting a tenancy model within an organization with each tenant representing a department comes with the disadvantage that it will prevent queries from spanning multiple departments. This proposal tries to overcome those limitations.</p><h2 id=alternatives-considered>Alternatives considered</h2><h3 id=aggregation-in-promql-api-clients>Aggregation in PromQL API clients</h3><p>In theory PromQL API clients could be aggregating/correlating query results from multiple tenants. For example Grafana could be used with multiple data sources and a cross tenant query could be achieved through using <a href=https://grafana.com/docs/grafana/latest/panels/transformations/>Transformations</a>.</p><p>As this approach comes with the following disadvantages, it was not considered further:</p><ul><li><p>Every PromQL API client needs to support the aggregation from various sources.</p></li><li><p>Queries that are written in PromQL can&rsquo;t be used without extra work across tenants.</p></li></ul><h3 id=multi-tenant-aggregation-in-the-query-frontends>Multi tenant aggregation in the query frontends</h3><p>Another approach to multi tenant query federation could be achieved by aggregation of partial query results within the query frontend. For this a query needs to be split into sub queries per tenant and afterwards the partial results need reduced into the final result.</p><p>The <a href=https://github.com/cortexproject/cortex/blob/f0c81bb59bf202db820403812e8dabcb64347bfd/pkg/querier/astmapper/parallel.go#L27>astmapper</a> package goes down a similar approach, but it cannot parallelize all query types. Ideally multi-tenant query federation should support the full PromQL language and the algorithms necessary would differ per query functions and operators used. This approach was deemed as a fairly complex way to achieve that tenant query federation.</p><h2 id=challenges>Challenges</h2><h3 id=aggregate-data-without-overlaps>Aggregate data without overlaps</h3><h4 id=challenge>Challenge</h4><p>The series in different tenants might have exactly the same labels and hence potentially collide which each other.</p><h4 id=proposal>Proposal</h4><p>In order to be able to always identify the tenant correctly, queries using multiple tenants should inject a tenant label named <code>__tenant_id__</code> and its value containing the tenant ID into the results. A potentially existing label with the same name should be stored in a label with the prefix <code>original_</code>.</p><p>Label selectors containing the tenant label should behave like any other labels. This can be achieved by selecting the tenants used in a multi tenant query.</p><h3 id=exposure-of-feature-to-the-user>Exposure of feature to the user</h3><h4 id=challenge-1>Challenge</h4><p>The tenant ID is currently read from the <code>X-Scope-OrgID</code> HTTP header. The tenant ID has those <a href=https://cortexmetrics.io/docs/guides/limitations/#tenant-id-naming>documented limitations</a> of values being allowed.</p><h4 id=proposal-1>Proposal</h4><p>For the query path a user should be able to specify a <code>X-Scope-OrgID</code> header with multiple tenant IDs. Multiple tenant IDs should then be propagating throughout out the system until it reaches the querier. The <code>Queryable</code> instance returned by the querier component, is wrapped by a <code>mergeQueryable</code>, which will aggregate the results from a <code>Queryable</code> per tenant and hence treated by the downstream components as a single tenant query.</p><p>To allow such queries to be processed we suggest that an experimental configuration flag <code>-querier.tenant-federation.enabled</code> will be added, which is switched off by default. Once enabled the value of the <code>X-Scope-OrgID</code> header should be interpreted as <code>|</code> separated list of tenant ids. Components which are not expecting multiple tenant ids (e.g. the ingress path) must signal an error if multiple are used.</p><h3 id=implementing-limits-fairness-and-observability-for-cross-tenant-queries>Implementing Limits, Fairness and Observability for Cross-Tenant queries</h3><h4 id=challenge-2>Challenge</h4><p>In Cortex the tenant id is used as the primary identifier for those components:</p><ul><li><p>The limits that apply to a certain query.</p></li><li><p>The query-frontend maintains a per tenant query queue to implement fairness.</p></li><li><p>Relevant metrics about the query are exposed under a <code>user</code> label.</p></li></ul><p>Having a query spanning multiple tenants, the existing methods are no longer correct.</p><h4 id=proposal-2>Proposal</h4><p>The identifier for aforementioned features for queries involving more than a single tenant should be derived from: An ordered, distinct list of tenant IDs, which is joined by a <code>|</code>. This will produce a reproducible identifier for the same set of tenants no matter which order they have been specified.</p><p>While this feature is considered experimental, this provides some insights and ability to limit multi-tenant queries with these short comings:</p><ul><li><p>Cardinality costs to the possible amount of tenant ID combinations.</p></li><li><p>Query limits applied to single tenants part of a multi tenant query are ignored.</p></li></ul><h2 id=conclusion>Conclusion</h2><table><thead><tr><th>Challenge</th><th>Status</th></tr></thead><tbody><tr><td>Aggregate data without overlap</td><td>Implementation in PR <a href=https://github.com/cortexproject/cortex/pull/3250>#3250</a></td></tr><tr><td>Exposure of feature to the user</td><td>Implementation in PR <a href=https://github.com/cortexproject/cortex/pull/3250>#3250</a></td></tr><tr><td>Implementing Limits, Fairness and Observability for Cross-Tenant queries</td><td>Implementation in PR <a href=https://github.com/cortexproject/cortex/pull/3250>#3250</a></td></tr></tbody></table><h3 id=future-work>Future work</h3><p>Those features are currently out of scope for this proposal, but we can foresee some interest implementing them after this proposal.</p><h4 id=cross-tenant-support-for-the-ruler>Cross Tenant support for the ruler</h4><p>Ability to use multi tenant queries in the ruler.</p><h4 id=allow-the-identifier-for-limits-fairness-and-observability-to-be-switched-out>Allow the identifier for limits, fairness and observability to be switched out</h4><p>It would be great if the source identifier could be made more pluggable. This could allow to for example base all of those features on another dimension (e.g. users rather than tenants)</p><h4 id=allow-customisation-of-the-label-used-to-expose-tenant-ids>Allow customisation of the label used to expose tenant ids</h4><p>As per this proposal the label name <code>__tenant_id__</code> is fixed, but users might want to be able to modify that through a configuration option.</p><h4 id=retain-overlapping-tenant-id-label-values-recursively>Retain overlapping tenant id label values recursively</h4><p>As per this proposal the tenant label injection retains an existing label value, but this is not implemented recursively. So if the result already contains <code>__tenant_id__</code> and <code>original__tenant_id__</code> labels, the value of the latter would be lost.</p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/proposals/documentation-versioning/index.html b/docs/proposals/documentation-versioning/index.html
index 45792ab935..102c2bc57f 100644
--- a/docs/proposals/documentation-versioning/index.html
+++ b/docs/proposals/documentation-versioning/index.html
@@ -10,7 +10,7 @@
 In this proposal, we want to solve this."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span class=td-sidebar-nav-active-item>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/version-documentation.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Documentation%20Versioning" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#problem>Problem</a></li><li><a href=#proposed-solution>Proposed solution</a><ul><li><a href=#example>Example:</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/documentation-versioning/ aria-disabled=true class="btn-link disabled">Documentation Versioning</a></li></ol></nav><div class=td-content><h1>Documentation Versioning</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/jaybatra26>Jay Batra</a></li><li>Date: March 2020</li><li>Status: proposal</li></ul><h2 id=problem>Problem</h2><p>In Cortex, currently, we are missing versioning of documentation. The idea is to have version documentation just like Prometheus.<a href=https://prometheus.io/docs/introduction/overview/><code>Prometheus</code></a>. Documentation is the main source of information for current contributors and first-timers. A properly versioned documentation will help everyone to have a proper place to look for answers before flagging it in the community.</p><p>In this proposal, we want to solve this. In particular, we want to:</p><ol><li>Version specific pages of the documentation</li><li>Include links to change version (the version must be in the URL)</li><li>Include the master version and last 3 minor releases. Documentation defaults to the last minor release.</li></ol><h2 id=proposed-solution>Proposed solution</h2><p>Currently, the documentation is residing under the docs/ folder of cortexproject/cortex. It is built by Hugo using the theme <a href=https://www.docsy.dev><code>docsy</code></a>. It will have a proper <a href=https://www.docsy.dev/docs/adding-content/versioning/#adding-a-version-drop-down-menu><code>drop-down menu</code></a> which will enable proper versioning. It has a section <a href=https://www.docsy.dev/docs/adding-content/versioning/#adding-a-version-drop-down-menu><code>params.version</code></a> in config.toml which will allow us to map URLs with proper versions. We will have to change all the occurrences of older doc links with new links. We will keep <code>master</code> version with 3 latest <code>release</code> versions. Each release is a minor version expressed as <code>1.x</code>. The document would default to latest minor version.</p><p>From the current doc, the following paths (and all their subpages) should be versioned for now:</p><ol><li><a href=https://cortexmetrics.io/docs/apis/>https://cortexmetrics.io/docs/apis/</a></li><li><a href=https://cortexmetrics.io/docs/configuration/>https://cortexmetrics.io/docs/configuration/</a> (moving v1.x Guarantees outside of the tree, because these shouldn&rsquo;t be versioned)</li></ol><p>The above should be versioned under a single URL path (<code>/docs/running-cortex/</code> in the following example, but final prefix is still to be decided).</p><h3 id=example>Example:</h3><p>For <code>master</code> version we would be able to use the above links via the following path</p><pre tabindex=0><code>/docs/running-cortex/master/configuration/
diff --git a/docs/proposals/generalize-modules/index.html b/docs/proposals/generalize-modules/index.html
index d3a54d5103..0d946dc1b5 100644
--- a/docs/proposals/generalize-modules/index.html
+++ b/docs/proposals/generalize-modules/index.html
@@ -8,7 +8,7 @@
 Cortex uses modules to start …"><meta property="og:title" content="Generalize Modules Service to make it extensible"><meta property="og:description" content="Author: @annanay25 Reviewers: @jtlisi, @pstibrany, @cyriltovena, @pracucci Date: April 2020 Status: Accepted Overview Cortex uses modules to start and operate services with dependencies. Inter-service dependencies are specified in a map and passed to a module manager which ensures that they are initialised in the right order of dependencies. While this works really well, the implementation is tied in specifically to the Cortex struct and is not flexible for use with other projects like Loki, which also require similar forms of dependency management."><meta property="og:type" content="article"><meta property="og:url" content="/docs/proposals/generalize-modules/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Generalize Modules Service to make it extensible"><meta itemprop=description content="Author: @annanay25 Reviewers: @jtlisi, @pstibrany, @cyriltovena, @pracucci Date: April 2020 Status: Accepted Overview Cortex uses modules to start and operate services with dependencies. Inter-service dependencies are specified in a map and passed to a module manager which ensures that they are initialised in the right order of dependencies. While this works really well, the implementation is tied in specifically to the Cortex struct and is not flexible for use with other projects like Loki, which also require similar forms of dependency management."><meta itemprop=wordCount content="580"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Generalize Modules Service to make it extensible"><meta name=twitter:description content="Author: @annanay25 Reviewers: @jtlisi, @pstibrany, @cyriltovena, @pracucci Date: April 2020 Status: Accepted Overview Cortex uses modules to start and operate services with dependencies. Inter-service dependencies are specified in a map and passed to a module manager which ensures that they are initialised in the right order of dependencies. While this works really well, the implementation is tied in specifically to the Cortex struct and is not flexible for use with other projects like Loki, which also require similar forms of dependency management."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span class=td-sidebar-nav-active-item>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/generalize-modules.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Generalize%20Modules%20Service%20to%20make%20it%20extensible" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#overview>Overview</a></li><li><a href=#specific-goals>Specific goals</a></li><li><a href=#proposed-design>Proposed Design</a><ul><li><a href=#modules-package>Modules package</a></li><li><a href=#changes-to-pkgcortex>Changes to <code>pkg/cortex</code>:</a></li></ul></li><li><a href=#usage>Usage</a><ul><li></li></ul></li><li><a href=#future-work>Future work</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/generalize-modules/ aria-disabled=true class="btn-link disabled">Generalize Modules Service to make it extensible</a></li></ol></nav><div class=td-content><h1>Generalize Modules Service to make it extensible</h1><header class=article-meta></header><ul><li>Author: @annanay25</li><li>Reviewers: @jtlisi, @pstibrany, @cyriltovena, @pracucci</li><li>Date: April 2020</li><li>Status: Accepted</li></ul><h2 id=overview>Overview</h2><p>Cortex uses modules to start and operate services with dependencies. Inter-service dependencies are specified in a map and passed to a module manager which ensures that they are initialised in the right order of dependencies. While this works really well, the implementation is tied in specifically to the Cortex struct and is not flexible for use with other projects like Loki, which also require similar forms of dependency management.</p><p>We would like to extend modules in cortex to a generic dependency management framework, that can be used by any project with no ties to cortex.</p><h2 id=specific-goals>Specific goals</h2><ul><li>Framework should allow for reusing cortex modules and allow us to:<ul><li>Add new modules</li><li>Overwrite the implementation of a current module</li><li>Manage dependencies</li></ul></li><li>Framework should allow for building an application from scratch using the <code>modules</code> package, with no dependencies on Cortex. For ex: Remove code from Loki that was copied from <code>pkg/cortex/cortex.go</code>.</li></ul><h2 id=proposed-design>Proposed Design</h2><h3 id=modules-package>Modules package</h3><p>To make the modules package extensible, we need to abstract away any Cortex specific details from the module manager. The proposed design is to:</p><ul><li>Make a new component <code>Manager</code>, which is envisioned to be a central manager for all modules of the application. It stores modules & dependencies, and will be housed under a new package <code>pkg/util/modules</code>. <code>Manager</code> has the following methods for interaction:</li></ul><pre tabindex=0><code>   func (m *Manager) RegisterModule(name string, initFn func() (Service, error))
diff --git a/docs/proposals/http-api-design/index.html b/docs/proposals/http-api-design/index.html
index cb5d8eb19c..026d4db182 100644
--- a/docs/proposals/http-api-design/index.html
+++ b/docs/proposals/http-api-design/index.html
@@ -8,7 +8,7 @@
 The purpose of this design …"><meta property="og:title" content="HTTP API Design"><meta property="og:description" content="Author: @jtlisi Reviewers: @pracucci, @pstibrany, @khaines, @gouthamve Date: March 2020 Status: Accepted Overview The purpose of this design document is to propose a set of standards that should be the basis of the Cortex HTTP API. This document will outline the current state of the Cortex http api and describe limitations that result from the current approach. It will also outline a set of paradigms on how http routes should be created within Cortex."><meta property="og:type" content="article"><meta property="og:url" content="/docs/proposals/http-api-design/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="HTTP API Design"><meta itemprop=description content="Author: @jtlisi Reviewers: @pracucci, @pstibrany, @khaines, @gouthamve Date: March 2020 Status: Accepted Overview The purpose of this design document is to propose a set of standards that should be the basis of the Cortex HTTP API. This document will outline the current state of the Cortex http api and describe limitations that result from the current approach. It will also outline a set of paradigms on how http routes should be created within Cortex."><meta itemprop=wordCount content="583"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="HTTP API Design"><meta name=twitter:description content="Author: @jtlisi Reviewers: @pracucci, @pstibrany, @khaines, @gouthamve Date: March 2020 Status: Accepted Overview The purpose of this design document is to propose a set of standards that should be the basis of the Cortex HTTP API. This document will outline the current state of the Cortex http api and describe limitations that result from the current approach. It will also outline a set of paradigms on how http routes should be created within Cortex."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span class=td-sidebar-nav-active-item>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/api_design.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=HTTP%20API%20Design" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#overview>Overview</a></li><li><a href=#current-design>Current Design</a></li><li><a href=#proposed-design>Proposed Design</a><ul><li><a href=#module-based-routing>Module-Based Routing</a></li><li><a href=#path-versioning>Path Versioning</a></li><li><a href=#backwards-compatibility>Backwards-Compatibility</a></li><li><a href=#implementation>Implementation</a></li><li><a href=#style>Style</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/http-api-design/ aria-disabled=true class="btn-link disabled">HTTP API Design</a></li></ol></nav><div class=td-content><h1>HTTP API Design</h1><header class=article-meta></header><ul><li>Author: @jtlisi</li><li>Reviewers: @pracucci, @pstibrany, @khaines, @gouthamve</li><li>Date: March 2020</li><li>Status: Accepted</li></ul><h2 id=overview>Overview</h2><p>The purpose of this design document is to propose a set of standards that should be the basis of the Cortex HTTP API. This document will outline the current state of the Cortex http api and describe limitations that result from the current approach. It will also outline a set of paradigms on how http routes should be created within Cortex.</p><h2 id=current-design>Current Design</h2><p>As things currently stand, the majority of HTTP API calls exist under the <code>/api/prom</code> path prefix. This prefix is configurable. However, since this prefix is shared between all the modules which leads to conflicts if the Alertmanager is attempted to be run as as part of the single binary (#1722).</p><h2 id=proposed-design>Proposed Design</h2><h3 id=module-based-routing>Module-Based Routing</h3><p>Cortex incorporates three separate APIs: Alertmanager, Prometheus, and Cortex. Each of these APIs should use a separate route prefix that accurately describes the API. Currently, all of the api calls in Cortex reside under the configured http prefix. Instead the following routing tree is proposed:</p><h4 id=prometheus><code>/prometheus/*</code></h4><p>Under this path prefix, Cortex will act as a Prometheus web server. It will host all of the required Prometheus api endpoints. For example to query cortex the endpoint <code>/prometheus/api/v1/query_range</code> will be used.</p><h4 id=alertmanager><code>/alertmanager/*</code></h4><p>Under this path prefix, Cortex will act as a Alertmanager web server. In this case, it will forward requests to the alertmanager and support the alertmanager API. This means for a user to access their Alertmanager UI, they will use the <code>/alertmanager</code> path of cortex.</p><h4 id=apiv1----the-cortex-api-will-exist-under-this-path-prefix><code>/api/v1/*</code> &ndash; The cortex API will exist under this path prefix.</h4><ul><li><code>/push</code></li><li><code>/chunks</code></li><li><code>/rules/*</code></li></ul><table><thead><tr><th>Current</th><th>Proposed</th></tr></thead><tbody><tr><td><code>/api/prom/push</code></td><td><code>/api/v1/push</code></td></tr><tr><td><code>/api/prom/chunks</code></td><td><code>/api/v1/chunks</code></td></tr><tr><td><code>/api/prom/rules/*</code></td><td><code>/api/v1/rules/*</code></td></tr></tbody></table><h4 id=service-endpoints>Service Endpoints</h4><p>A number of endpoints currently exist that are not under the <code>/api/prom</code> prefix that provide basic web interfaces and trigger operations for cortex services. These endpoints will all be placed under a url with their service name as a prefix if it is applicable.</p><table><thead><tr><th>Current</th><th>Proposed</th></tr></thead><tbody><tr><td><code>/status</code></td><td><code>/multitenant-alertmanager/status</code></td></tr><tr><td><code>/config</code></td><td><code>/config</code></td></tr><tr><td><code>/ring</code></td><td><code>/ingester/ring</code></td></tr><tr><td><code>/ruler_ring</code></td><td><code>/ruler/ring</code></td></tr><tr><td><code>/compactor/ring</code></td><td><code>/compactor/ring</code></td></tr><tr><td><code>/store-gateway/ring</code></td><td><code>/store-gateway/ring</code></td></tr><tr><td><code>/ha-tracker</code></td><td><code>/distributor/ha_tracker</code></td></tr><tr><td><code>/all_user_stats</code></td><td><code>/distributor/all_user_stats</code></td></tr><tr><td><code>/user_stats</code></td><td><code>/distributor/user_stats</code></td></tr><tr><td><code>/flush</code></td><td><code>/ingester/flush</code></td></tr><tr><td><code>/shutdown</code></td><td><code>/ingester/shutdown</code></td></tr></tbody></table><h3 id=path-versioning>Path Versioning</h3><p>Cortex will utilize path based versioning similar to both Prometheus and Alertmanager. This will allow future versions of the API to be released with changes over time.</p><h3 id=backwards-compatibility>Backwards-Compatibility</h3><p>The new API endpoints and the current http prefix endpoints can be maintained concurrently. The flag to configure these endpoints will be maintained as <code>http.prefix</code>. This will allow us to roll out the new API without disrupting the current routing schema. The original http prefix endpoints can maintained indefinitely or be phased out over time. Deprecation warnings can be added to the current API either when initialized or utilized. This can be accomplished by injecting a middleware that logs a warning whenever a legacy API endpoint is used.</p><p>In cases where Cortex is run as a single binary, the Alertmanager module will only be accessible using the new API.</p><h3 id=implementation>Implementation</h3><p>This will be implemented by adding an API module to the Cortex service. This module will handle setting up all the required HTTP routes with Cortex. It will be designed around a set of interfaces required to fulfill the API. This is similar to how the <code>v1</code> Prometheus API is implemented.</p><h3 id=style>Style</h3><ul><li>All new paths will utilize <code>_</code> instead of <code>-</code> for their url to conform with Prometheus and its use of the underscore in the <code>query_range</code> endpoint. This applies to all operations endpoints. Component names in the path can still contain dashes. For example: <code>/store-gateway/ring</code>.</li></ul></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/proposals/index.html b/docs/proposals/index.html
index 15f13e4ec7..c9fb733c78 100644
--- a/docs/proposals/index.html
+++ b/docs/proposals/index.html
@@ -1,7 +1,7 @@
 <!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.101.0"><link rel=alternate type=application/rss+xml href=/docs/proposals/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/favicons/android-96x96.png sizes=96x96><link rel=icon type=image/png href=/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/favicons/android-192x192.png sizes=192x192><title>Proposals | Cortex</title><meta name=description content><meta property="og:title" content="Proposals"><meta property="og:description" content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta property="og:type" content="website"><meta property="og:url" content="/docs/proposals/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Proposals"><meta itemprop=description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Proposals"><meta name=twitter:description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span class=td-sidebar-nav-active-item>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/_index.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Proposals" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/ aria-disabled=true class="btn-link disabled">Proposals</a></li></ol></nav><div class=td-content><h1>Proposals</h1><header class=article-meta></header><ul><li><a href=/docs/proposals/auth-gateway/>Authentication Gateway</a></li><li><a href=/docs/proposals/blocks-storage-bucket-index/>Blocks storage bucket index</a></li><li><a href=/docs/proposals/blocks-storage-sharding/>Blocks storage sharding</a></li><li><a href=/docs/proposals/cross-tenant-query-federation/>Cross-Tenant Query Federation</a></li><li><a href=/docs/proposals/tenant-deletion/>Deletion of Tenant Data from Blocks Storage</a></li><li><a href=/docs/proposals/documentation-versioning/>Documentation Versioning</a></li><li><a href=/docs/proposals/generalize-modules/>Generalize Modules Service to make it extensible</a></li><li><a href=/docs/proposals/http-api-design/>HTTP API Design</a></li><li><a href=/docs/proposals/ingesters-migration/>Migrating ingesters from chunks to blocks and back.</a></li><li><a href=/docs/proposals/parallel-compaction/>Parallel Compaction by Time Interval</a></li><li><a href=/docs/proposals/tenant-retention/>Retention of Tenant Data from Blocks Storage</a></li><li><a href=/docs/proposals/ring-multikey/>Ring Multikey</a></li><li><a href=/docs/proposals/ruler-ha/>Ruler HA</a></li><li><a href=/docs/proposals/ruler-high-availability/>Ruler High Availability</a></li><li><a href=/docs/proposals/ruler-tenant-federation/>Ruler Tenant Federation</a></li><li><a href=/docs/proposals/scalable-alertmanager/>Scalable Alertmanager</a></li><li><a href=/docs/proposals/scalable-query-frontend/>Scalable Query Frontend</a></li><li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/>Shuffle sharding and zone awareness</a></li><li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/>Shuffle sharding on the read path</a></li><li><a href=/docs/proposals/support-metadata-api/>Support metadata API</a></li><li><a href=/docs/proposals/block-storage-time-series-deletion/>Time Series Deletion from Blocks Storage</a></li><li><a href=/docs/proposals/timeseries-partitioning-in-compactor/>Timeseries Partitioning in Compactor</a></li></ul></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/proposals/ingesters-migration/index.html b/docs/proposals/ingesters-migration/index.html
index e65210fece..ff0c9a461b 100644
--- a/docs/proposals/ingesters-migration/index.html
+++ b/docs/proposals/ingesters-migration/index.html
@@ -11,7 +11,7 @@
 Introduction This short document describes the first step in full migration of the Cortex cluster from using chunks storage to using blocks storage, specifically switching ingesters to using blocks, and modification of queriers to query both chunks and blocks storage."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span class=td-sidebar-nav-active-item>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/ingesters-migration.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Migrating%20ingesters%20from%20chunks%20to%20blocks%20and%20back." target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#warning>Warning</a></li><li><a href=#introduction>Introduction</a></li><li><a href=#ingesters>Ingesters</a></li><li><a href=#querying>Querying</a></li><li><a href=#work-to-do>Work to do</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/ingesters-migration/ aria-disabled=true class="btn-link disabled">Migrating ingesters from chunks to blocks and back.</a></li></ol></nav><div class=td-content><h1>Migrating ingesters from chunks to blocks and back.</h1><header class=article-meta></header><ul><li>Author: @pstibrany</li><li>Reviewers:</li><li>Date: June 2020</li><li>Status: Replaced with <a href=https://github.com/cortexproject/cortex/blob/v1.11.1/docs/blocks-storage/migrate-from-chunks-to-blocks.md>migration guide (now removed from this site)</a>.</li></ul><h2 id=warning>Warning</h2><p>Suggestions from this proposal were implemented, but general procedure outlined here doesn&rsquo;t quite work in
diff --git a/docs/proposals/parallel-compaction/index.html b/docs/proposals/parallel-compaction/index.html
index f630c6f6f9..2874a09bb1 100644
--- a/docs/proposals/parallel-compaction/index.html
+++ b/docs/proposals/parallel-compaction/index.html
@@ -8,7 +8,7 @@
 As a part of pushing Cortex’s scaling capability at AWS, we have done performance …"><meta property="og:title" content="Parallel Compaction by Time Interval"><meta property="og:description" content="Author: Roy Chiang Date: May 2021 Status: Proposed Introduction As a part of pushing Cortex’s scaling capability at AWS, we have done performance testing with Cortex and found the compactor to be one of the main limiting factors for higher active timeseries limit per tenant. The documentation Compactor describes the responsibilities of a compactor, and this proposal focuses on the limitations of the current compactor architecture. In the current architecture, compactor has simple sharding, meaning that a single tenant is sharded to a single compactor."><meta property="og:type" content="article"><meta property="og:url" content="/docs/proposals/parallel-compaction/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Parallel Compaction by Time Interval"><meta itemprop=description content="Author: Roy Chiang Date: May 2021 Status: Proposed Introduction As a part of pushing Cortex’s scaling capability at AWS, we have done performance testing with Cortex and found the compactor to be one of the main limiting factors for higher active timeseries limit per tenant. The documentation Compactor describes the responsibilities of a compactor, and this proposal focuses on the limitations of the current compactor architecture. In the current architecture, compactor has simple sharding, meaning that a single tenant is sharded to a single compactor."><meta itemprop=wordCount content="1044"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Parallel Compaction by Time Interval"><meta name=twitter:description content="Author: Roy Chiang Date: May 2021 Status: Proposed Introduction As a part of pushing Cortex’s scaling capability at AWS, we have done performance testing with Cortex and found the compactor to be one of the main limiting factors for higher active timeseries limit per tenant. The documentation Compactor describes the responsibilities of a compactor, and this proposal focuses on the limitations of the current compactor architecture. In the current architecture, compactor has simple sharding, meaning that a single tenant is sharded to a single compactor."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span class=td-sidebar-nav-active-item>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/parallel-compaction.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Parallel%20Compaction%20by%20Time%20Interval" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#introduction>Introduction</a><ul><li><a href=#problem-and-requirements>Problem and Requirements</a></li></ul></li><li><a href=#design>Design</a><ul><li><a href=#parallelize-work>Parallelize Work</a></li></ul></li><li><a href=#scenarios>Scenarios</a><ul><li><a href=#bad-block-resulting-in-non-ideal-compaction-groups>Bad block resulting in non-ideal compaction groups</a></li></ul></li><li><a href=#alternatives>Alternatives</a><ul><li><a href=#shard-compaction-jobs-amongst-compactors-with-a-scheduler>Shard compaction jobs amongst compactors with a scheduler</a></li><li><a href=#contribute-to-thanos-for-a-more-scalable-compactor>Contribute to Thanos for a more scalable compactor</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/parallel-compaction/ aria-disabled=true class="btn-link disabled">Parallel Compaction by Time Interval</a></li></ol></nav><div class=td-content><h1>Parallel Compaction by Time Interval</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/roystchiang>Roy Chiang</a></li><li>Date: May 2021</li><li>Status: Proposed</li></ul><hr><h2 id=introduction>Introduction</h2><p>As a part of pushing Cortex’s scaling capability at AWS, we have done performance testing with Cortex and found the compactor to be one of the main limiting factors for higher active timeseries limit per tenant. The documentation <a href=https://cortexmetrics.io/docs/blocks-storage/compactor/#how-compaction-works>Compactor</a> describes the responsibilities of a compactor, and this proposal focuses on the limitations of the current compactor architecture. In the current architecture, compactor has simple sharding, meaning that a single tenant is sharded to a single compactor. The compactor generates compaction groups, which are groups of Prometheus TSDB blocks that can be compacted together, independently of another group. However, a compactor currently handles compaction groups of a single tenant iteratively, meaning that blocks belonging non-overlapping times are not compacted in parallel.</p><p>Cortex ingesters are responsible for uploading TSDB blocks with data emitted by a tenant. These blocks are considered as level-1 blocks, as they contain duplicate timeseries for the same time interval, depending on the replication factor. <a href=https://cortexmetrics.io/docs/blocks-storage/compactor/#how-compaction-works>Vertical compaction</a> is done to merge all the blocks with the same time interval and deduplicate the samples. These merged blocks are level-2 blocks. Subsequent compactions such as horizontal compaction can happen, further increasing the compaction level of the blocks.</p><h3 id=problem-and-requirements>Problem and Requirements</h3><p>Currently, a compactor is able to compact up to 20M timeseries within 2 hours for a level-2 compaction, including the time to download blocks, compact, and upload the newly compacted block. We would like to increase the timeseries limit per tenant, and compaction is one of the limiting factors. In addition, we would like to achieve the following:</p><ul><li>Compact multiple non-overlapping time intervals concurrently, so we can achieve higher throughput for the compaction of a single tenant</li><li>We should be able to scale up, down compactor as needed, depending on how many compactions are pending</li><li>Insight into the compaction progress of a tenant, such as the number of compactions required in order to catch up to the newest blocks</li></ul><h2 id=design>Design</h2><p>We accept the fact that a single compaction can potentially take more than 2 hours to compact, and we achieve higher compaction throughput through horizontally scaling the compactor. To compact more blocks in parallel for a single tenant, we distribute the compaction groups to compactors, instead of introducing more parallelism within a compactor.</p><h3 id=parallelize-work>Parallelize Work</h3><p>This proposal builds heavily on top of the <a href=https://github.com/cortexproject/cortex/pull/2616>GrafanaLabs approach of introducing parallelism via time intervals</a>. The difference being that a single tenant is now sharded across multiple compactors instead of just a single compactor. The initial approach will be to work on distinct time intervals, but the compactor planner can be later extended to introduce parallelism within a time interval as well.</p><p>The following is an example of parallelize work at each level:</p><p><img src=/images/proposals/parallel-compaction-grouping.png alt="Parallel Compaction Grouping"></p><p>Compactors are shuffle-sharded, meaning that 1 tenant can belong to multiple compactors, and these subset of compactors determine which blocks should be compacted together. Compactors determine amongst themselves the responsibility of the compaction blocks, by using a hash of time interval and tenant id, and putting it on the sharding ring.</p><p>The benefit of this approach is that this aligns with what Cortex currently does in Ruler. The downside is that a compaction job can only be assigned to a single compactor, rather than all of the compactors sharded for the tenant. If a compaction job takes forever, other tenants sharded to the same compactor will be blocked until the issue is resolved. With the scheduler approach, any compactor assigned to a given tenant can pick up any work required.</p><p><img src=/images/proposals/parallel-compaction-without-scheduler.png alt="Parallel Compaction Without Scheduler"></p><h2 id=scenarios>Scenarios</h2><h3 id=bad-block-resulting-in-non-ideal-compaction-groups>Bad block resulting in non-ideal compaction groups</h3><p>A Cortex operator configures the compaction block range as 2h and 6h. If a full 6-hour block cannot be compacted due to compaction failures, the compactor should not split up the group into subgroups, as this may cause suboptimal grouping of block. Cortex has full information regarding all the available blocks, so we should utilize this information to achieve the best compaction group possible.</p><h2 id=alternatives>Alternatives</h2><h3 id=shard-compaction-jobs-amongst-compactors-with-a-scheduler>Shard compaction jobs amongst compactors with a scheduler</h3><p><img src=/images/proposals/parallel-compaction-design.png alt="Parallel Compaction Architecture"></p><p>We add a new component Compactor Scheduler, which is responsible for calculating the compaction plan, and distributing compaction groups to compactors. The planner is sharded by tenant id, so that we can horizontally scale the planner as needed in order to accept more tenants in the cortex cluster. A tenant will have two queues inside the planner, a compaction queue and a clean up queue, similar to how the query frontend currently holds queues of pending queries.</p><p>Once a compactor scheduler pushes a job to a compactor, the job is no longer available. Every set interval, or once the compaction is done, a compactor will update the compactor schedule the current status of the compaction job. If a compactor does not provide an update to the scheduler within a timeout, the compaction job becomes available to be assigned to other compactors.</p><h4 id=concurrency>Concurrency</h4><p>To achieve concurrency within a single tenant, compactor scheduler will push jobs to compactors. Compactors are shuffle-sharded by tenant id, to prevent a large tenant from impacting the compaction of other tenants. Compactor will download blocks from long term storage, compact, and upload. Compactor will also pull from the clean up queues from scheduler, and delete blocks marked for deletion.</p><h4 id=consistency>Consistency</h4><p>On resharding of compactor schedulers, a tenant might move to a different scheduler. We can either drop the current compactor job in order to prevent duplicate compaction jobs, or continue compaction. I propose that the compactor drops the compaction job if the compaction group no longer belongs to the original compactor scheduler. This way, we do not have duplicate compactions happening, and we can minimize work wasted.</p><h3 id=contribute-to-thanos-for-a-more-scalable-compactor>Contribute to Thanos for a more scalable compactor</h3><p>Instead of introducing parallelism on the Cortex compactor level, we move the parallelism to the Thanos compactor itself. Thanos has a <a href=https://docs.google.com/document/d/1xi0V8DB0hE54XgkogJRnNL6yH7C5JThJywlLFoC6dCQ/>proposal to make compactor more scalable</a>, and a <a href=https://github.com/thanos-io/thanos/pull/3807>PR</a>. Cortex will enjoy higher throughput per tenant if Thanos is able to speed up the compaction, and we can keep the Cortex architecture the same. However, this approach means that a single tenant is still sharded to a single compactor. In order to compact more groups at once, we must scale up compactor vertically. Although vertical scaling can get us far, we should scale horizontally where we can.</p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/proposals/ring-multikey/index.html b/docs/proposals/ring-multikey/index.html
index 592a90a533..e6a36b3b1b 100644
--- a/docs/proposals/ring-multikey/index.html
+++ b/docs/proposals/ring-multikey/index.html
@@ -7,7 +7,7 @@
 Cortex implements a ring structure to share information of registered pods for …"><meta property="og:title" content="Ring Multikey"><meta property="og:description" content="Author: Daniel Blando Date: August 2022 Status: Proposed Background Cortex implements a ring structure to share information of registered pods for each service. The data stored and used by the ring need to be implemented via Codec interface. Currently, the only supported Codec to the ring is Desc. Desc is a proto.Message with a list of instances descriptions. It is used to store the data for each pod and saved on a supported KV store."><meta property="og:type" content="article"><meta property="og:url" content="/docs/proposals/ring-multikey/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Ring Multikey"><meta itemprop=description content="Author: Daniel Blando Date: August 2022 Status: Proposed Background Cortex implements a ring structure to share information of registered pods for each service. The data stored and used by the ring need to be implemented via Codec interface. Currently, the only supported Codec to the ring is Desc. Desc is a proto.Message with a list of instances descriptions. It is used to store the data for each pod and saved on a supported KV store."><meta itemprop=wordCount content="727"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Ring Multikey"><meta name=twitter:description content="Author: Daniel Blando Date: August 2022 Status: Proposed Background Cortex implements a ring structure to share information of registered pods for each service. The data stored and used by the ring need to be implemented via Codec interface. Currently, the only supported Codec to the ring is Desc. Desc is a proto.Message with a list of instances descriptions. It is used to store the data for each pod and saved on a supported KV store."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span class=td-sidebar-nav-active-item>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/ring-multikey.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Ring%20Multikey" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#background>Background</a></li><li><a href=#problem>Problem</a></li><li><a href=#proposal>Proposal</a><ul><li><a href=#multikey-interface>Multikey interface</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/ring-multikey/ aria-disabled=true class="btn-link disabled">Ring Multikey</a></li></ol></nav><div class=td-content><h1>Ring Multikey</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/danielblando>Daniel Blando</a></li><li>Date: August 2022</li><li>Status: Proposed</li></ul><h2 id=background>Background</h2><p>Cortex implements a ring structure to share information of registered pods for each
diff --git a/docs/proposals/ruler-ha/index.html b/docs/proposals/ruler-ha/index.html
index de82aaf14b..dfd6bd439d 100644
--- a/docs/proposals/ruler-ha/index.html
+++ b/docs/proposals/ruler-ha/index.html
@@ -12,7 +12,7 @@
 This proposal consolidates multiple existing PRs from the AWS team working on this feature, as well as future work needed to complete support. The hope is that a more holistic view will make for more productive discussion and review of the individual changes, as well as provide better tracking of overall progress."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span class=td-sidebar-nav-active-item>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/ruler-ha.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Ruler%20HA" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#introduction>Introduction</a></li><li><a href=#problem>Problem</a></li><li><a href=#proposal>Proposal</a><ul><li><a href=#make-replicationfactor-configurable>Make ReplicationFactor configurable</a></li><li><a href=#adjustments-for-alerts_for_state>Adjustments for ALERTS_FOR_STATE</a></li><li><a href=#weak-and-strong-quorum-in-listrules-and-listalerts>Weak and Strong Quorum in ListRules and ListAlerts</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/ruler-ha/ aria-disabled=true class="btn-link disabled">Ruler HA</a></li></ol></nav><div class=td-content><h1>Ruler HA</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/soonping-amzn>Soon-Ping Phang</a></li><li>Date: June 2022</li><li>Status: Deprecated</li></ul><hr><h2 id=introduction>Introduction</h2><p><em>This proposal is deprecated in favor of the new <a href=/docs/proposals/ruler-high-availability/>proposal</a></em></p><p>This proposal consolidates multiple existing PRs from the AWS team working on this feature, as well as future work needed to complete support. The hope is that a more holistic view will make for more productive discussion and review of the individual changes, as well as provide better tracking of overall progress.</p><p>The original issue is <a href=https://github.com/cortexproject/cortex/issues/4435>#4435</a>.</p><h2 id=problem>Problem</h2><p>Rulers in Cortex currently run with a replication factor of 1, wherein each RuleGroup is assigned to exactly 1 ruler. This lack of redundancy creates the following risks:</p><ul><li>Rule group evaluation<ul><li>Missed evaluations due to a ruler outage, possibly caused by a deployment, noisy neighbour, hardware failure, etc.</li><li>Missed evaluations due to a ruler brownout due to other tenant rule groups sharing the same ruler (noisy neighbour)</li></ul></li><li>API
diff --git a/docs/proposals/ruler-high-availability/index.html b/docs/proposals/ruler-high-availability/index.html
index 809d9e9bce..537f0c36fa 100644
--- a/docs/proposals/ruler-high-availability/index.html
+++ b/docs/proposals/ruler-high-availability/index.html
@@ -11,7 +11,7 @@
 Rule group evaluation Missed evaluations due to a ruler outage, possibly caused by a deployment, noisy neighbour, hardware failure, etc. Missed evaluations due to a ruler brownout due to other tenant rule groups sharing the same ruler (noisy neighbour) API Inconsistent API results during resharding (e."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span class=td-sidebar-nav-active-item>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/ruler-ha-new.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Ruler%20High%20Availability" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#problem>Problem</a></li><li><a href=#proposal>Proposal</a><ul><li><a href=#make-replicationfactor-configurable>Make ReplicationFactor configurable</a></li><li><a href=#prometheus-change>Prometheus change</a></li><li><a href=#api-ha>API HA</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/ruler-high-availability/ aria-disabled=true class="btn-link disabled">Ruler High Availability</a></li></ol></nav><div class=td-content><h1>Ruler High Availability</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/rajagopalanand>Anand Rajagopal</a></li><li>Date: Aug 2024</li><li>Status: Proposed</li></ul><hr><h2 id=problem>Problem</h2><p>Rulers in Cortex currently run with a replication factor of 1, wherein each RuleGroup is assigned to exactly 1 ruler. This lack of redundancy creates the following risks:</p><ul><li>Rule group evaluation<ul><li>Missed evaluations due to a ruler outage, possibly caused by a deployment, noisy neighbour, hardware failure, etc.</li><li>Missed evaluations due to a ruler brownout due to other tenant rule groups sharing the same ruler (noisy neighbour)</li></ul></li><li>API<ul><li>Inconsistent API results during resharding (e.g. due to a deployment) when rulers are in a transition state loading rule groups</li></ul></li></ul><p>This proposal attempts to mitigate the above risks by enabling a ruler replication factor of greater than 1, allowing multiple rulers to evaluate the same rule group — effectively.</p><h2 id=proposal>Proposal</h2><h3 id=make-replicationfactor-configurable>Make ReplicationFactor configurable</h3><p>ReplicationFactor in Ruler is currently hardcoded to 1. Making this a configurable parameter is the first step to enabling HA in ruler. The parameter value will be 1 by default. To enable Ruler HA for rule group evaluation, a new flag will be created</p><p>A replication factor greater than 1 will result in the following</p><ul><li>Ring will pick R rulers for a rule group where R=RF</li><li>The primary ruler (R1), when active, will take ownership of the rule group</li><li>Non-primary ruler R2 will check if R1 is active. If R1 is not active, R2 will take ownership of the rule group</li><li>Non-primary ruler R3 (if RF=3) will check if R1 and R2 are active. If they are both inactive/unhealthy, then R3 will take owership of the rule group</li><li>Non-primary rulers will drop their ownership when R1 becomes active after an outage</li></ul><p>With this redundancy, the maximum duration of missed evaluations will be limited to the sync interval of the rule groups, reducing the impact of primary Ruler unavailability.</p><h3 id=prometheus-change>Prometheus change</h3><p>No Prometheus change is required for this proposal</p><h3 id=api-ha>API HA</h3><p>An interim solution is addressed in this <a href=https://github.com/cortexproject/cortex/issues/5773>#5773</a> PR. This will be modified such that the replicas will return both active and passive rule groups and the API handler will continue to de-duplicate the results.
diff --git a/docs/proposals/ruler-tenant-federation/index.html b/docs/proposals/ruler-tenant-federation/index.html
index 253bff378b..53fb48a226 100644
--- a/docs/proposals/ruler-tenant-federation/index.html
+++ b/docs/proposals/ruler-tenant-federation/index.html
@@ -7,7 +7,7 @@
 This document aims to describe how to implement the ability to allow rules to …"><meta property="og:title" content="Ruler Tenant Federation"><meta property="og:description" content="Author: Rees Dooley Date: November 2021 Status: Accepted Overview This document aims to describe how to implement the ability to allow rules to cover data from more than a single Cortex tenant, here after referred to as federated rules. Since currently rules are owned by, query data from and save resulting series in the same tenant, this document aims to provide clear delineation of who owns a federated rule, what tenants the federated rule queries data from and where the federated rule saves resulting series."><meta property="og:type" content="article"><meta property="og:url" content="/docs/proposals/ruler-tenant-federation/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Ruler Tenant Federation"><meta itemprop=description content="Author: Rees Dooley Date: November 2021 Status: Accepted Overview This document aims to describe how to implement the ability to allow rules to cover data from more than a single Cortex tenant, here after referred to as federated rules. Since currently rules are owned by, query data from and save resulting series in the same tenant, this document aims to provide clear delineation of who owns a federated rule, what tenants the federated rule queries data from and where the federated rule saves resulting series."><meta itemprop=wordCount content="617"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Ruler Tenant Federation"><meta name=twitter:description content="Author: Rees Dooley Date: November 2021 Status: Accepted Overview This document aims to describe how to implement the ability to allow rules to cover data from more than a single Cortex tenant, here after referred to as federated rules. Since currently rules are owned by, query data from and save resulting series in the same tenant, this document aims to provide clear delineation of who owns a federated rule, what tenants the federated rule queries data from and where the federated rule saves resulting series."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span class=td-sidebar-nav-active-item>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/federated-ruler.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Ruler%20Tenant%20Federation" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#overview>Overview</a></li><li><a href=#reasoning>Reasoning</a></li><li><a href=#challenges>Challenges</a><ul><li><a href=#allow-federated-rules-behind-feature-flag>Allow federated rules behind feature flag</a></li><li><a href=#allow-federated-rules-only-for-select-tenants>Allow federated rules only for select tenants</a></li><li><a href=#where-to-store-resulting-series-of-federated-rule>Where to store resulting series of federated rule</a></li><li><a href=#which-tenants-to-query-from-for-federated-rules>Which tenants to query from for federated rules</a></li></ul></li><li><a href=#conclusion>Conclusion</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/ruler-tenant-federation/ aria-disabled=true class="btn-link disabled">Ruler Tenant Federation</a></li></ol></nav><div class=td-content><h1>Ruler Tenant Federation</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/rdooley>Rees Dooley</a></li><li>Date: November 2021</li><li>Status: Accepted</li></ul><h2 id=overview>Overview</h2><p>This document aims to describe how to implement the ability to allow rules to cover data from more than a single Cortex tenant, here after referred to as federated rules. Since currently rules are owned by, query data from and save resulting series in the same tenant, this document aims to provide clear delineation of who owns a federated rule, what tenants the federated rule queries data from and where the federated rule saves resulting series.</p><p>A federated rule is any rule which contains the <code>src_tenants</code> field.</p><h2 id=reasoning>Reasoning</h2><p>The primary use case for allowing federated rules which query data from multiple tenants is the administration of cortex.</p><p>In the case of the administration of cortex, when running Cortex within a large organization, there may be metrics spanning across tenants which might be desired to be monitored e.g. administrative metrics of the cortex system like <code>prometheus_rule_evaluation_failures_total</code> aggregated by <code>__tenant_id__</code>. In this case, a team e.g. <code>infra</code> may wish to be able to create a rule, owned by <code>infra</code> which queries multiple tenants <code>t0|t1|...|ti</code> and stores resulting series in <code>infra</code>.</p><h2 id=challenges>Challenges</h2><h3 id=allow-federated-rules-behind-feature-flag>Allow federated rules behind feature flag</h3><h4 id=challenge>Challenge</h4><p>Federated tenant rules and alerts will not be a good fit for organization and should be behind a feature flag.</p><h4 id=proposal>Proposal</h4><p>For federated rules, creation of federated rules (those sourcing data from multiple tenants) should be blocked behind the feature flag <code>ruler.enable-federated-rules</code></p><p>If tenant federation is enabled, then ruler should use a <code>mergeQueryable</code> to aggregate the results of querying multiple tenants.</p><h3 id=allow-federated-rules-only-for-select-tenants>Allow federated rules only for select tenants</h3><h4 id=challenge-1>Challenge</h4><p>For many organizations, the ability for any tenant to write a rule querying any other tenant is not acceptable and more fine grained control is required</p><h4 id=proposal-1>Proposal</h4><p>Since the current default is that a tenant should only be able to write rules against itself, we suggest a config option <code>ruler.allowed-federated-tenants</code>, a string slice of OrgIDs like <code>infra</code> or <code>0|1|2|3|4</code> which are allowed to write rules against all tenants. If a tenant <code>bar</code> attempts to create a federated rule, an error should be returned by the ruler api. Similarly an option <code>ruler.disallowed-federated-tenants</code> explicitly states a list of tenants for which federated rules are not allowed. Combining these in a <code>util.AllowedTenants</code> should allow one to quickly determine if federation is enabled or disabled for a given tenant at rule creation.</p><h3 id=where-to-store-resulting-series-of-federated-rule>Where to store resulting series of federated rule</h3><h4 id=challenge-2>Challenge</h4><p>A single tenant rule always stores produced series in the tenant where the rule exists. This 1 -> 1 mapping becomes a many -> 1 mapping for federated rules.</p><h4 id=proposal-2>Proposal</h4><p>Tenants owning a federated rule the resulting series is saved in the tenant which owns the rule.</p><h3 id=which-tenants-to-query-from-for-federated-rules>Which tenants to query from for federated rules</h3><h4 id=challenge-3>Challenge</h4><p>A single tenant rule always queries the tenant which owns the rule. This 1 -> 1 mapping becomes a 1 -> many mapping for federated rules.</p><h4 id=proposal-3>Proposal</h4><p>As some use cases will demand that a specific federated rule, querying tenant B and C, is stored in the owning teams tenant A, an option to allow explicit assignment of source tenants for a federated rule is needed.</p><p>To support this we suggest an additional field <code>src_tenants</code> on the rule group containing an array of OrgIDs e.g. <code>[t0,t1,...,ti]</code> which when present determines which tenants to query for the given rule. Rule group is chosen as it reduces repetition between rules.</p><h2 id=conclusion>Conclusion</h2><table><thead><tr><th>Challenge</th><th>Status</th></tr></thead><tbody><tr><td>Allow federated rules behind feature flag</td><td>Planned but not yet implemented</td></tr><tr><td>Allow federated rules only for select tenants</td><td>Planned but not yet implemented</td></tr><tr><td>Where to store resulting series of federated rules</td><td>Planned but not yet implemented</td></tr><tr><td>Which tenants to query from for federated rules</td><td>Planned but not yet implemented</td></tr></tbody></table></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/proposals/scalable-alertmanager/index.html b/docs/proposals/scalable-alertmanager/index.html
index f384c70bcb..53bfa4dbe9 100644
--- a/docs/proposals/scalable-alertmanager/index.html
+++ b/docs/proposals/scalable-alertmanager/index.html
@@ -7,7 +7,7 @@
 The Cortex Alertmanager at its current state supports …"><meta property="og:title" content="Scalable Alertmanager"><meta property="og:description" content="Author: Josh Abreu Date: December 2020 Status: Accepted Context and Background The Cortex Alertmanager at its current state supports high-availability by using the same technique as the upstream Prometheus Alertmanager: we gossip silences and notifications between replicas to achieve eventual consistency. This allows us to tolerate machine failure without interruption of service. The caveat here is traffic between the ruler and Alertmanagers must not be load balanced as alerts themselves are not gossiped."><meta property="og:type" content="article"><meta property="og:url" content="/docs/proposals/scalable-alertmanager/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Scalable Alertmanager"><meta itemprop=description content="Author: Josh Abreu Date: December 2020 Status: Accepted Context and Background The Cortex Alertmanager at its current state supports high-availability by using the same technique as the upstream Prometheus Alertmanager: we gossip silences and notifications between replicas to achieve eventual consistency. This allows us to tolerate machine failure without interruption of service. The caveat here is traffic between the ruler and Alertmanagers must not be load balanced as alerts themselves are not gossiped."><meta itemprop=wordCount content="1040"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Scalable Alertmanager"><meta name=twitter:description content="Author: Josh Abreu Date: December 2020 Status: Accepted Context and Background The Cortex Alertmanager at its current state supports high-availability by using the same technique as the upstream Prometheus Alertmanager: we gossip silences and notifications between replicas to achieve eventual consistency. This allows us to tolerate machine failure without interruption of service. The caveat here is traffic between the ruler and Alertmanagers must not be load balanced as alerts themselves are not gossiped."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span class=td-sidebar-nav-active-item>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/scalable-alertmanager.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Scalable%20Alertmanager" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#context-and-background>Context and Background</a></li><li><a href=#problem-and-requirements>Problem and Requirements</a></li><li><a href=#design>Design</a><ul><li><a href=#routing--sharding>Routing & Sharding</a></li><li><a href=#persistence--state>Persistence & State</a></li><li><a href=#replication--consistency>Replication & Consistency</a></li><li><a href=#architecture>Architecture</a></li></ul></li><li><a href=#conclusion>Conclusion</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/scalable-alertmanager/ aria-disabled=true class="btn-link disabled">Scalable Alertmanager</a></li></ol></nav><div class=td-content><h1>Scalable Alertmanager</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/gotjosh>Josh Abreu</a></li><li>Date: December 2020</li><li>Status: Accepted</li></ul><h2 id=context-and-background>Context and Background</h2><p>The Cortex Alertmanager at its current state supports high-availability by using the same technique as the upstream Prometheus Alertmanager: we gossip silences and notifications between replicas to achieve eventual consistency. This allows us to tolerate machine failure without interruption of service. The caveat here is traffic between the ruler and Alertmanagers must not be load balanced as alerts themselves are not gossiped.</p><p>By itself it is not horizontally scalable; it is recommended to run a maximum of 3 replicas (for high availability). Each alertmanager replica will contain an internal instance for every tenant. The Alertmanager uses roughly ~3.7MB of memory and ~0.001 CPU cores per tenant, with an average tenant having ~10 active alerts and ~1 silences. These stats were measured from a cluster with ~80 QPS (Alerts received Per Second), ~40 NPS (Notifications Per Second), and ~700 configurations over a single replica.</p><h2 id=problem-and-requirements>Problem and Requirements</h2><p>Current numbers show a reasonably sized machine can handle 2000 tenants in the current service. We would like to be able to scale this up to 10x without increasing the machine size; i.e. we would like to make Cortex Alertmanager service horizontally scalable.</p><p>Furthermore, we desire to preserve the following characteristics:</p><ul><li>A single Cortex Alertmanager replica crashing or exiting abruptly should not cause externally-visible downtime or failure to deliver notifications.</li><li>Users should have an eventually consistent view of all the alerts current firing and under active notification, favouring availability of the Cortex Alertmanager over consistency.</li><li>We should be able to scale up, down and roll out new versions of the service without any service interruption or data loss.</li></ul><h2 id=design>Design</h2><p>This is a big design, and as such we have divided the problem up into 4 areas: Routing & Sharding, Persistence & State, Replication & Consistency, and Architecture. The options and solutions I propose are divided along these lines to aid clarity.</p><h3 id=routing--sharding>Routing & Sharding</h3><p>To achieve horizontal scalability, we need to distribute the workload among replicas of the service. We need to choose an appropriate field to use to distribute the workload. The field must be present on all the API requests to the Alertmanager service.</p><p><strong>We propose the sharding on Tenant ID</strong>. The simplicity of this implementation, would allow us to get up and running relatively quickly whilst helping us validate assumptions. We intend to use the existing ring code to manage this. Other options such as tenant ID + receiver or Tenant ID + route are relatively complex as distributor components (in this case the Ruler) would need to be aware of Alertmanager configuration.</p><h3 id=persistence--state>Persistence & State</h3><p>Alertmanager is a stateful service; it stores the notification state and configured silences. By default, Alertmanager persists its state to disk every 15mins. In the horizontally scalable Alertmanager service, we need to move this state around as the number of replicas grows and shrinks. We also need to persist this state across restarts and rolling upgrades.</p><p><strong>We propose</strong> making each Alertmanager replica flush the state to object storage, under its own key that’s a combination of tenant ID + replica periodically. This state on durable storage will only be used when cold-starting the cluster.</p><p>This mechanism covers multiple challenges (scaling up & down, rolling restarts, total cluster failure). <strong>To top this off, in the implementation, we’ll always try to request state from other replicas in the cluster before ever trying to go to object storage.</strong></p><h3 id=replication--consistency>Replication & Consistency</h3><p>Upstream Alertmanager replicates notification state between replicas to ensure notifications are not sent more than once. Cortex Alertmanager does the same. When we move to a model where AM instances for a given tenant only live on a subset of replicas, we have to decide how we will keep these replicas in sync.</p><p><strong>We have an option of doing nothing</strong> and use the existing gossip mechanism and gossip all tenants state to all replicas. The AM Router will then drop state updates for tenants which don’t shard to a given replica. I think this will be easy to implement as it requires few changes but probably won’t scale.</p><p><strong>However, I propose we Synchronized state over gRPC.</strong> The upstream Alertmanager notification dispatcher uses a timeout-based approach for its notifications. Using the “peer order”, it’ll wait a certain amount of time before letting other replicas know if this notification succeeded or not. If it didn’t, the next replica in line will try to send the notification. We propose to communicate this via gRPC calls.</p><p>While I propose the use of gRPC, is good to note that doing nothing is still a solid candidate for consideration but not having to maintain and operate two separate replication patterns feels like a win.</p><h3 id=architecture>Architecture</h3><p>To implement the sharding strategy we need a component in charge of handling incoming alerts and API requests, distributing it to the corresponding shard.</p><p>The idea here is to have an “Alertmanager Distributor” as a first stop in the reception of alerts. Once alerts are received, the component is in charge of validating the alerts against the limits. Validated alerts are then sent to multiple managers in parallel.</p><p>The individual pieces of this component (sharding, limits) cannot be optional - the optional part of it is where we decide to run it.</p><p>We can either run it as a separate service or embed it. <strong>I propose we simply embed it</strong>. At its core it’ll be simpler to operate. With future work making it possible to run as a separate service so that operators can scale when/if needed.</p><h2 id=conclusion>Conclusion</h2><p>Under the assumption we implement the options proposed above, our architecture looks like this:</p><p><img src=/images/proposals/scalable-am.png alt="Scalable Alertmanager Architecture"></p><p><strong>POST /api/v1/alerts (from the ruler) can go to any Alertmanager replica.</strong> The AM distributor uses the ring to write alerts to a quorum of AM managers (reusing the existing code). We continue to use the same in-memory data structure from the upstream Alertmanager to save alerts and notify other pieces</p><p><strong>GET /api/v1/alerts and /api/v1/alerts/group</strong> uses the ring to find the right alertmanager replicas for the given tenant. They read from a quorum of alertmanager replicas and return an union of the results.</p><p><strong>Alertmanager state is replicated between instances to keep them in sync</strong>. Where the state is replicated to to is controlled by the ring.</p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/proposals/scalable-query-frontend/index.html b/docs/proposals/scalable-query-frontend/index.html
index 13d2a2b54f..2359d47841 100644
--- a/docs/proposals/scalable-query-frontend/index.html
+++ b/docs/proposals/scalable-query-frontend/index.html
@@ -7,7 +7,7 @@
 This document aims to describe the role that the Cortex Query Frontend plays in …"><meta property="og:title" content="Scalable Query Frontend"><meta property="og:description" content="Author: Joe Elliott Date: April 2020 Status: Proposed Overview This document aims to describe the role that the Cortex Query Frontend plays in running multitenant Cortex at scale. It also describes the challenges of horizontally scaling the query frontend component and includes several recommendations and options for creating a reliably scalable query-frontend. Finally, we conclude with a discussion of the overall philosophy of the changes and propose an alternative."><meta property="og:type" content="article"><meta property="og:url" content="/docs/proposals/scalable-query-frontend/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content="docs"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Scalable Query Frontend"><meta itemprop=description content="Author: Joe Elliott Date: April 2020 Status: Proposed Overview This document aims to describe the role that the Cortex Query Frontend plays in running multitenant Cortex at scale. It also describes the challenges of horizontally scaling the query frontend component and includes several recommendations and options for creating a reliably scalable query-frontend. Finally, we conclude with a discussion of the overall philosophy of the changes and propose an alternative."><meta itemprop=wordCount content="2290"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Scalable Query Frontend"><meta name=twitter:description content="Author: Joe Elliott Date: April 2020 Status: Proposed Overview This document aims to describe the role that the Cortex Query Frontend plays in running multitenant Cortex at scale. It also describes the challenges of horizontally scaling the query frontend component and includes several recommendations and options for creating a reliably scalable query-frontend. Finally, we conclude with a discussion of the overall philosophy of the changes and propose an alternative."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span class=td-sidebar-nav-active-item>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/scalable-query-frontend.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Scalable%20Query%20Frontend" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#overview>Overview</a></li><li><a href=#reasoning>Reasoning</a></li><li><a href=#query-frontend-role>Query Frontend Role</a><ul><li><a href=#load-shedding>Load Shedding</a></li><li><a href=#query-retries>Query Retries</a></li><li><a href=#shardingparallelization>Sharding/Parallelization</a></li><li><a href=#query-alignmentcaching>Query Alignment/Caching</a></li><li><a href=#tenancy-fairness>Tenancy Fairness</a></li></ul></li><li><a href=#challenges-and-proposals>Challenges And Proposals</a><ul><li><a href=#dynamic-querier-concurrency>Dynamic Querier Concurrency</a></li><li><a href=#overwhelming-promql-concurrency>Overwhelming PromQL Concurrency</a></li><li><a href=#increased-time-to-failure>Increased Time To Failure</a></li><li><a href=#querier-discovery-lag>Querier Discovery Lag</a></li><li><a href=#dilutes-tenant-fairness>Dilutes Tenant Fairness</a></li></ul></li><li><a href=#alternative>Alternative</a></li><li><a href=#conclusion>Conclusion</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/scalable-query-frontend/ aria-disabled=true class="btn-link disabled">Scalable Query Frontend</a></li></ol></nav><div class=td-content><h1>Scalable Query Frontend</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/joe-elliott>Joe Elliott</a></li><li>Date: April 2020</li><li>Status: Proposed</li></ul><h2 id=overview>Overview</h2><p>This document aims to describe the <a href=#query-frontend-role>role</a> that the Cortex Query Frontend plays in running multitenant Cortex at scale. It also describes the <a href=#challenges-and-proposals>challenges</a> of horizontally scaling the query frontend component and includes several recommendations and options for creating a reliably scalable query-frontend. Finally, we conclude with a discussion of the overall philosophy of the changes and propose an <a href=#alternative>alternative</a>.</p><p>For the original design behind the query frontend, you should read <a href=https://docs.google.com/document/d/1lsvSkv0tiAMPQv-V8vI2LZ8f4i9JuTRsuPI_i-XcAqY>Cortex Query Optimisations design doc from 2018-07</a>.</p><h2 id=reasoning>Reasoning</h2><p>Query frontend scaling is becoming increasingly important for two primary reasons.</p><p>The Cortex team is working toward a scalable single binary solution. Recently the query-frontend was <a href=https://github.com/cortexproject/cortex/pull/2437>added</a> to the Cortex single binary mode and, therefore, needs to seamlessly scale. Technically, nothing immediately breaks when scaling the query-frontend, but there are a number of concerns detailed in <a href=#challenges-and-proposals>Challenges And Proposals</a>.</p><p>As the query-frontend continues to <a href=https://github.com/cortexproject/cortex/pull/1878>support additional features</a> it will start to become a bottleneck of the system. Current wisdom is to run very few query-frontends in order to maximize <a href=#tenancy-fairness>Tenancy Fairness</a> but as more features are added scaling horizontally will become necessary.</p><h2 id=query-frontend-role>Query Frontend Role</h2><h3 id=load-shedding>Load Shedding</h3><p>The query frontend maintains a queue per tenant of configurable length (default 100) in which it stores a series of requests from that tenant. If this queue fills up then the frontend will return 429’s thus load shedding the rest of the system.</p><p>This is particularly effective due to the “pull” based model in which queriers pull requests from query frontends.</p><h3 id=query-retries>Query Retries</h3><p>The query frontend is capable of retrying a query on another querier if the first should fail due to OOM or network issues.</p><h3 id=shardingparallelization>Sharding/Parallelization</h3><p>The query frontend shards requests by interval and <a href=https://github.com/cortexproject/cortex/pull/1878>other factors</a> to concurrently run a single query across multiple queriers.</p><h3 id=query-alignmentcaching>Query Alignment/Caching</h3><p>Queries are aligned to their own step and then stored/retrieved from cache.</p><h3 id=tenancy-fairness>Tenancy Fairness</h3><p>By maintaining one queue per tenant, a low demand tenant will have the same opportunity to have a query serviced as a high demand tenant. See <a href=#dilutes-tenant-fairness>Dilutes Tenant Fairness</a> for additional discussion.</p><p>For clarity, tenancy fairness only comes into play when queries are actually being queued in the query frontend. Currently this rarely occurs, but as <a href=https://github.com/cortexproject/cortex/pull/1878>query sharding</a> becomes more aggressive this may become the norm.</p><h2 id=challenges-and-proposals>Challenges And Proposals</h2><h3 id=dynamic-querier-concurrency>Dynamic Querier Concurrency</h3><h4 id=challenge>Challenge</h4><p>For every query frontend the querier adds a <a href=https://github.com/cortexproject/cortex/blob/50f53dba8f8bd5f62c0e85cc5d85684234cd1c1c/pkg/querier/frontend/worker.go#L146>configurable number of goroutines</a> which are each capable of executing a query. Therefore, scaling the query frontend impacts the amount of work each individual querier is attempting to do at any given time.</p><p>Scaling up may cause a querier to attempt more work than they are configured for due to restrictions such as memory and cpu limits. Additionally, the promql engine itself is limited in the number of queries it can do as configured by the <code>-querier.max-concurrent</code> parameter. Attempting more queries concurrently than this value causes the queries to queue up in the querier itself.</p><p>For similar reasons scaling down the query frontend may cause a querier to not use its allocated memory and cpu effectively. This will lower effective resource utilization. Also, because individual queriers will be doing less work, this may cause increased queueing in the query frontends.</p><h4 id=proposal>Proposal</h4><p>Currently queriers are configured to have a <a href=https://github.com/cortexproject/cortex/blob/50f53dba8f8bd5f62c0e85cc5d85684234cd1c1c/pkg/querier/frontend/worker.go#L146>max parallelism per query frontend</a>. An additional “total max concurrency” flag should be added.</p><p>Total Max Concurrency would then be evenly divided amongst all available query frontends. This would decouple the amount of work a querier is attempting to do with the number of query frontends that happen to exist at this moment. Consequently this would allow allocated resources (e.g. k8s cpu/memory limits) to remain balanced with the work the querier was attempting as the query frontend is scaled up or down.</p><p>A <a href=https://github.com/cortexproject/cortex/pull/2456>PR</a> has already been merged to address this.</p><h3 id=overwhelming-promql-concurrency>Overwhelming PromQL Concurrency</h3><h4 id=challenge-1>Challenge</h4><p>If #frontends > promql concurrency then the queriers are incapable of devoting even a single worker to each query frontend without risking queueing in the querier. Queuing in the querier is a highly undesirable state and one of the primary reasons the query frontend was originally created.</p><h4 id=proposal-1>Proposal</h4><p>When #frontends > promql concurrency then each querier will maintain <a href=https://github.com/cortexproject/cortex/blob/8fb86155a7c7c155b8c4d31b91b267f9631b60ba/pkg/querier/frontend/worker.go#L194-L200>exactly one connection</a> to every frontend. As the query frontend is <a href=https://github.com/cortexproject/cortex/blob/8fb86155a7c7c155b8c4d31b91b267f9631b60ba/pkg/querier/frontend/frontend.go#L279-L332>currently coded</a> it will attempt to use every open GRPC connection to execute a query in the attached queriers. Therefore, in this situation where #frontends > promql concurrency, the querier is exposing itself to more work then it is actually configured to perform.</p><p>To prevent this we will add “flow control” information to the <a href=https://github.com/cortexproject/cortex/blob/master/pkg/querier/frontend/frontend.proto#L21>ProcessResponse message</a> that is used to return query results from the querier to the query frontend. In an active system this message is passed multiple times per second from the queriers to the query frontends and would be a reliable way for the frontends to track the state of queriers and balance load.</p><p>There are a lot of options for an exact implementation of this idea. An effective solution should be determined and chosen by modeling a set of alternatives. The details of this would be included in another design doc. A simple implementation would look something like the following:</p><p>Add two new fields to <a href=https://github.com/cortexproject/cortex/blob/master/pkg/querier/frontend/frontend.proto#L21>ProcessResponse</a>:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-protobuf data-lang=protobuf><span style=display:flex><span><span style=color:#204a87;font-weight:700>message</span> <span style=color:#000>ProcessResponse</span> <span style=color:#000;font-weight:700>{</span><span style=color:#a40000>
diff --git a/docs/proposals/shuffle-sharding-and-zone-awareness/index.html b/docs/proposals/shuffle-sharding-and-zone-awareness/index.html
index 06c2d37be9..9297c650b1 100644
--- a/docs/proposals/shuffle-sharding-and-zone-awareness/index.html
+++ b/docs/proposals/shuffle-sharding-and-zone-awareness/index.html
@@ -10,7 +10,7 @@
 Given every tenant writes series to all ingesters, there’s no isolation between tenants - a single misbehaving tenant can affect the whole cluster."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span class=td-sidebar-nav-active-item>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/shuffle-sharding-and-zone-awareness.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Shuffle%20sharding%20and%20zone%20awareness" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#shuffle-sharding-and-zone-awareness>Shuffle sharding and zone awareness</a><ul><li><a href=#background>Background</a></li><li><a href=#goal>Goal</a></li><li><a href=#proposal>Proposal</a></li><li><a href=#guaranteed-properties>Guaranteed properties</a></li><li><a href=#proof-of-concept>Proof of concept</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ aria-disabled=true class="btn-link disabled">Shuffle sharding and zone awareness</a></li></ol></nav><div class=td-content><h1>Shuffle sharding and zone awareness</h1><header class=article-meta></header><ul><li>Author: @pracucci, @tomwilkie, @pstibrany</li><li>Reviewers:</li><li>Date: August 2020</li><li>Status: Accepted, implemented in <a href=https://github.com/cortexproject/cortex/pull/3090>PR #3090</a></li></ul><h2 id=shuffle-sharding-and-zone-awareness>Shuffle sharding and zone awareness</h2><h3 id=background>Background</h3><p>Cortex shards the received series across all available ingesters. In a multi-tenant cluster, each tenant series are sharded across all ingesters. This allows to horizontally scale the series across the pool of ingesters but also suffers some issues:</p><ol><li>Given every tenant writes series to all ingesters, there’s no isolation between tenants - a single misbehaving tenant can affect the whole cluster.</li><li>Each ingester needs an open TSDB per tenant per ingester - which has significant memory overhead. The larger the number of tenants, the higher the TSDB memory overhead, regardless of the number of series stored in each TSDB.</li><li>Similarly, the number of uploaded blocks to the storage every 2 hours is a function of the number of TSDBs open for each ingester. A cluster with a large number of small tenants will upload a very large number of blocks to the storage, each block being very small, increasing the number of API calls against the storage bucket.</li></ol><p>Cortex currently supports sharding a tenant to a subset of the ingesters on the write path <a href=https://github.com/cortexproject/cortex/pull/1947>PR</a>, using a feature called “<strong>subring</strong>”. However, the current subring implementation suffers two issues:</p><ol><li><strong>No zone awareness:</strong> it doesn’t guarantee selected instances are balanced across availability zones</li><li><strong>No shuffling:</strong> the implementation is based on the hash ring and it selects N consecutive instances in the ring. This means that, instead of minimizing the likelihood that two tenants share the same instances, it emphasises it. In order to provide a good isolation between tenants, we want to minimize the chances that two tenants share the same instances.</li></ol><h3 id=goal>Goal</h3><p>The goal of this work is to fix “shuffling” and “zone-awareness” when building the subring for a given tenant, honoring the following properties:</p><ul><li><strong>Stability:</strong> given the same ring, the algorithm always generates the same subring for a given tenant, even across different machines</li><li><a href=https://en.wikipedia.org/wiki/Consistent_hashing><strong>Consistency:</strong></a> when the ring is resized, only n/m series are remapped on average (where n is the number of series and m is the number of replicas).</li><li><strong>Shuffling:</strong> probabilistically and for a large enough cluster, ensure every tenant gets a different set of instances, with a reduced number of overlapping instances between two tenants to improve failure isolation.</li><li><strong>Zone-awareness (balanced):</strong> the subring built for each tenant contains a balanced number of instances for each availability zone. Selecting the same number of instances in each zone is an important property because we want to preserve the balance of in-memory series across ingesters. Having less replicas in one zone will mean more load per node in this zone, which is something we want to avoid.</li></ul><h3 id=proposal>Proposal</h3><p>This proposal is based on <a href=https://aws.amazon.com/builders-library/workload-isolation-using-shuffle-sharding/>Amazon’s Shuffle Sharding article</a> and the algorithm has been inspired by shuffle sharding implementation in the <a href=https://github.com/awslabs/route53-infima/blob/master/src/main/java/com/amazonaws/services/route53/infima/SimpleSignatureShuffleSharder.java>AWS Route53 infima library</a>.</p><p>Given a tenant and a shard size S (number of instances to which tenant data/workload should be sharded to), we build a subring selecting N instances from each zone, where N = ceil(S / num of zones). The shard size S is required to be a multiple of the number of zones, in order to select an equal number of instances from each zone.</p><p>To do it, we <strong>treat each zone as a separate ring</strong> and select N unique instances from each zone. The instances selection process works as follow:</p><ol><li>Generate a seed based on the tenant ID</li><li>Initialise a pseudo random number generator with the tenant’s seed. The random generator must guarantee predictable numbers given the same input seed.</li><li>Generate a sequence of N random numbers, where N is the number of instances to select from the zone. Each random number is used as a “token” to look up instances in the ring. For each random number:</li><li>Lookup the instance holding that token in the ring</li><li>If the instance has not been previously selected, then pick it</li><li>If the instance was previously selected (we call this a “collision”), then continue walking the ring clockwise until we find an instance which has not been selected yet</li></ol><h3 id=guaranteed-properties>Guaranteed properties</h3><h4 id=stability>Stability</h4><p>The same tenant ID always generates the same seed. Given the same seed, the pseudo number random generator always generates the same sequence of numbers.</p><p>This guarantees that, given the same ring, we generate the same exact subring for a given tenant.</p><h4 id=consistency>Consistency</h4><p>The consistency property is honored by two aspects of the algorithm:</p><ol><li>The quantity of random numbers generated is always equal to the shard size S, even in case of “collisions”. A collision is when the instance holding the random token has already been picked and we need to select a different instance which has not been picked yet.</li><li>In case of collisions, we select the “next” instance continuing walking the ring instead of generating another random number</li></ol><h5 id=example-adding-an-instance-to-the-ring>Example adding an instance to the ring</h5><p>Let’s consider an initial ring with 3 instances and 1 zone (for simplicity):</p><ul><li>I1 - Tokens: 1, 8, 15</li><li>I2 - Tokens: 5, 11, 19</li><li>I3 - Tokens: 7, 13, 21</li></ul><p>With a replication factor = 2, the random sequence looks up:</p><ul><li>3 (I2)</li><li>6 (I1)</li></ul><p>Then we add a new instance and the <strong>updated ring</strong> is:</p><ul><li>I1 - Tokens: 1, 8, 15</li><li>I2 - Tokens: 5, 11, 19</li><li>I3 - Tokens: 7, 13, 21</li><li>I4 - Tokens: 4, 7, 17</li></ul><p>Now, let’s compare two different algorithms to solve collisions:</p><ul><li>Using the random generator:<br>Random sequence = 3 (<strong>I4</strong>), 6 (I4 - collision), 12 (<strong>I3</strong>)<br><strong>all instances are different</strong> (I4, I3)</li><li>Walking the ring:<br>Random sequence = 3 (<strong>I4</strong>), 6 (I4 - collision, next is <strong>I1</strong>)<br><strong>only 1 instance is different</strong> (I4, I1)</li></ul><h4 id=shuffling>Shuffling</h4><p>Unless when resolving collisions, the algorithm doesn’t walk the ring to find the next instances, but uses a sequence of random numbers. This guarantees instances are shuffled, between different tenants, when building the subring.</p><h4 id=zone-awareness>Zone-awareness</h4><p>We treat each zone as a separate ring and select an equal number of instances from each zone. This guarantees a fair balance of instances between zones.</p><h3 id=proof-of-concept>Proof of concept</h3><p>We’ve built a <a href=https://github.com/cortexproject/cortex/pull/3090>reference implementation</a> of the proposed algorithm, to test the properties described above.</p><p>In particular, we’ve observed that the <a href=https://github.com/cortexproject/cortex/pull/3090/files#diff-121ffce90aa9932f6b87ffd138e0f36aR281>actual distribution</a> of matching instances between different tenants is very close to the <a href=https://docs.google.com/spreadsheets/d/1FXbiWTXi6bdERtamH-IfmpgFq1fNL4GP_KX_yJvbRi4/edit>theoretical one</a>, as well as consistency and stability properties are both honored.</p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/proposals/shuffle-sharding-on-the-read-path/index.html b/docs/proposals/shuffle-sharding-on-the-read-path/index.html
index d1825ea954..47fb2d166f 100644
--- a/docs/proposals/shuffle-sharding-on-the-read-path/index.html
+++ b/docs/proposals/shuffle-sharding-on-the-read-path/index.html
@@ -14,7 +14,7 @@
 This approach is similar to the techniques described in Amazon’s Shuffle Sharding article, but currently suffers from a non random selection of nodes (proposed solution below)."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span class=td-sidebar-nav-active-item>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/shuffle-sharding-on-the-read-path.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Shuffle%20sharding%20on%20the%20read%20path" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#background>Background</a></li><li><a href=#goals>Goals</a><ul><li><a href=#prerequisite-fix-subring-shuffling>Prerequisite: fix subring shuffling</a></li><li><a href=#query-frontend--queriers-shuffle-sharding>Query-frontend → Queriers shuffle sharding</a></li><li><a href=#how-querier-runs-query-frontend-jobs>How querier runs query-frontend jobs</a></li><li><a href=#proposal-1>Proposal</a></li><li><a href=#properties>Properties</a></li><li><a href=#implementation-notes>Implementation notes</a></li><li><a href=#configuration>Configuration</a></li><li><a href=#evaluated-alternatives>Evaluated alternatives</a></li></ul></li><li><a href=#querier--store-gateway-shuffle-sharding>Querier → Store-gateway shuffle sharding</a><ul><li><a href=#introduction>Introduction</a></li><li><a href=#proposal-2>Proposal</a></li><li><a href=#configuration-1>Configuration</a></li><li><a href=#implementation-notes-1>Implementation notes</a></li><li><a href=#evaluated-alternatives-1>Evaluated alternatives</a></li></ul></li><li><a href=#querier-ingesters-shuffle-sharding>Querier→ Ingesters shuffle sharding</a><ul><li><a href=#problem>Problem</a></li><li><a href=#proposal-use-only-the-information-contained-in-the-ring>Proposal: use only the information contained in the ring.</a></li><li><a href=#considered-alternative-1-ingesters-expose-list-of-tenants>Considered alternative #1: Ingesters expose list of tenants</a></li><li><a href=#considered-alternative-2-streaming-updates-from-ingesters-to-queriers>Considered alternative #2: streaming updates from ingesters to queriers</a></li></ul></li><li><a href=#ruler-sharding>Ruler sharding</a><ul><li><a href=#introduction-1>Introduction</a></li><li><a href=#proposal-4>Proposal</a></li><li><a href=#other-notes>Other notes</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ aria-disabled=true class="btn-link disabled">Shuffle sharding on the read path</a></li></ol></nav><div class=td-content><h1>Shuffle sharding on the read path</h1><header class=article-meta></header><ul><li>Author: @pracucci, @tomwilkie, @pstibrany</li><li>Reviewers:</li><li>Date: August 2020</li><li>Status: Proposed, partially implemented</li></ul><h2 id=background>Background</h2><p>Cortex currently supports sharding of tenants to a subset of the ingesters on the write path <a href=https://github.com/cortexproject/cortex/pull/1947>PR</a>.</p><p>This feature is called “subring”, because it computes a subset of nodes registered to the hash ring. The aim of this feature is to improve isolation between tenants and reduce the number of tenants impacted by an outage.</p><p>This approach is similar to the techniques described in <a href=https://aws.amazon.com/builders-library/workload-isolation-using-shuffle-sharding/>Amazon’s Shuffle Sharding article</a>, but currently suffers from a non random selection of nodes (<em>proposed solution below</em>).</p><p>Cortex can be <strong>configured</strong> with a default subring size, and then it can be <a href=https://cortexmetrics.io/docs/configuration/configuration-file/#limits_config>customized on a per-tenant basis</a>. The per-tenant configuration is live reloaded during runtime and applied without restarting the Cortex process.</p><p>The subring sharding currently supports only the write-path. The read-path is not shuffle sharding aware. For example, an outage of more than one ingester with RF=3 will affect all tenants, or a particularly noisy tenant wrt queries has the ability to affect all tenants.</p><h2 id=goals>Goals</h2><p>The Cortex <strong>read path should support shuffle sharding to isolate</strong> the impact of an outage in the cluster. The shard size must be dynamically configurable on a per-tenant basis during runtime.</p><p>This deliverable involves introducing shuffle sharding in:</p><ul><li><strong>Query-frontend → Querier</strong> (for queries sharding) <a href=https://github.com/cortexproject/cortex/pull/3113>PR #3113</a></li><li><strong>Querier → Store-gateway</strong> (for blocks sharding) <a href=https://github.com/cortexproject/cortex/pull/3069>PR #3069</a></li><li><strong>Querier→ Ingesters</strong> (for queries on recent data)</li><li><strong>Ruler</strong> (for rule and alert evaluation)</li></ul><h3 id=prerequisite-fix-subring-shuffling>Prerequisite: fix subring shuffling</h3><p>The solution is implemented in <a href=https://github.com/cortexproject/cortex/pull/3090>https://github.com/cortexproject/cortex/pull/3090</a>.</p><h4 id=the-problem>The problem</h4><p>The subring is a subset of nodes that should be used for a specific tenant.</p><p>The current subring implementation doesn’t shuffle tenants across nodes. Given a tenant ID, it finds the first node owning the hash(tenant ID) token and then it picks N distinct consecutive nodes walking the ring clockwise.</p><p>For example, in a cluster with 6 nodes (numbered 1-6) and a replication factor of 3, three tenants (A, B, C) could have the following shards:</p><table><thead><tr><th>Tenant ID</th><th>Node 1</th><th>Node 2</th><th>Node 3</th><th>Node 4</th><th>Node 5</th><th>Node 6</th></tr></thead><tbody><tr><td>A</td><td>x</td><td>x</td><td>x</td><td></td><td></td><td></td></tr><tr><td>B</td><td></td><td>x</td><td>x</td><td>x</td><td></td><td></td></tr><tr><td>C</td><td></td><td></td><td>x</td><td>x</td><td>x</td><td></td></tr></tbody></table><h4 id=proposal>Proposal</h4><p>We propose to build the subring picking N distinct and random nodes registered in the ring, using the following algorithm:</p><ol><li>SID = tenant ID</li><li>SID = hash(SID)</li><li>Look for the node owning the token range containing FNV-1a(SID)</li><li>Loop to (2) until we’ve found N distinct nodes (where N is the shard size)</li></ol><p><em>hash() function to be decided. The required property is to be strong enough to not generate loops across multiple subsequent hashing of the previous hash.</em></p><h3 id=query-frontend--queriers-shuffle-sharding>Query-frontend → Queriers shuffle sharding</h3><p>Implemented in <a href=https://github.com/cortexproject/cortex/pull/3113>https://github.com/cortexproject/cortex/pull/3113</a>.</p><h3 id=how-querier-runs-query-frontend-jobs>How querier runs query-frontend jobs</h3><p>Today <strong>each</strong> querier connects to <strong>each</strong> query-frontend instance, and calls a single “Process” method via gRPC.</p><p>“Process” is a bi-directional streaming gRPC method – using the server-to-client stream for sending requests from query-frontend to the querier, and client-to-server stream for returning results from querier to the query-frontend. NB this is the opposite of what might be considered normal. Query-frontend scans all its queues with pending query requests, and picks a query to execute based on a fair schedule between tenants.</p><p>The query request is then sent to an idle querier worker over the stream opened in the Process method, and the query-frontend then waits for a response from querier. This loop repeats until querier disconnects.</p><h3 id=proposal-1>Proposal</h3><p>To support shuffle sharding, Query-Frontends will keep a list of connected Queriers, and randomly (but consistently between query-frontends) choose N of them to distribute requests to. When Query-Frontend looks for the next request to send to a given querier, it will only consider tenants that “belong” to the Querier.</p><p>To choose N Queriers for a tenant, we propose to use a simple algorithm:</p><ol><li>Sort all Queriers by their ID</li><li>SID = tenant ID</li><li>SID = hash(SID)</li><li>Pick the querier from the list of sorted queries with:<br>index = FNV-1a(SID) % number of Queriers</li><li>Loop to (3) until we’ve found N distinct queriers (where N is the shard size) and stop early if there aren’t enough queriers</li></ol><p><em>hash() function to be decided. The required property is to be strong enough to not generate loops across multiple subsequent hashing of the previous hash.</em></p><h3 id=properties>Properties</h3><ul><li><strong>Stability:</strong> this will produce the same result on all query-frontends as long as all queriers are connected to all query-frontends.</li><li><strong>Simplicity:</strong> no external dependencies.</li><li><strong>No consistent hashing:</strong> adding/removing queriers will cause “resharding” of tenants between queriers. While in general that’s not desirable property, queriers are stateless so it doesn’t seem to matter in this case.</li></ul><h3 id=implementation-notes>Implementation notes</h3><ul><li><strong>Caching:</strong> once this list of queriers to use for a tenant is computed in the query-frontend, it is cached in memory until queriers are added or removed. Per-tenant cache entries will have a TTL to discard tenants not “seen” since a while.</li><li><strong>Querier ID:</strong> Query-frontends currently don’t have any identity for queriers. We need to introduce sending of a unique ID (eg. hostname) by querier to query-frontend when it calls “Process” method.</li><li><strong>Backward-compatibility:</strong> when querier shuffle sharding is enabled, the system expects that both query-frontend and querier will run a compatible version. Cluster version upgrade will require to rollout new query-frontends and queriers first, and then enable shuffle sharding.</li><li><strong>UI:</strong> we propose to expose the current state of the query-frontend through a new endpoint which should display:<ul><li>Which querier are connected to the query-frontend</li><li>Are there any “old” queriers, that are receiving requests from all tenants?</li><li>Mapping of tenants to queriers. Note that this mapping may only be available for tenants with pending requests on given query-frontend, and therefore be very dynamic.</li></ul></li></ul><h3 id=configuration>Configuration</h3><ul><li><strong>Shard size</strong> will be configurable on a per-tenant basis via existing “runtime-configuration” mechanism (limits overrides). Changing a value for a tenant needs to invalidate cached per-tenant queriers.</li><li>Queriers shard size will be a different setting than then one used for writes.</li></ul><h3 id=evaluated-alternatives>Evaluated alternatives</h3><h4 id=use-the-subring>Use the subring</h4><p>An alternative option would be using the subring. This implies having queriers registering to the hash ring and query-frontend instances using the ring client to find the queriers subring for each tenant.</p><p>This solution looks adding more complexity without any actual benefit.</p><h4 id=change-query-frontend--querier-architecture>Change query-frontend → querier architecture</h4><p>Completely different approach would be to introduce a place where starting queriers would register (eg. DNS-based service discovery), and let query-frontends discover queriers from this central registry.</p><p>Possible benefit would be that queriers don’t need to initiate connection to all query-frontends, but query-frontends would only connect to queriers for which they have actual pending requests. However this would be a significant redesign of how query-frontend / querier communication works.</p><h2 id=querier--store-gateway-shuffle-sharding>Querier → Store-gateway shuffle sharding</h2><p>Implemented in <a href=https://github.com/cortexproject/cortex/pull/3069>https://github.com/cortexproject/cortex/pull/3069</a>.</p><h3 id=introduction>Introduction</h3><p>As of today, the store-gateway supports blocks sharding with customizable replication factor (defaults to 3). Blocks of a single tenant are sharded across all store-gateway instances and so to execute a query the querier may touch any store-gateway in the cluster.</p><p>The current sharding implementation is based on a <strong>hash ring</strong> formed by store-gateway instances.</p><h3 id=proposal-2>Proposal</h3><p>The proposed solution to add shuffle sharding support to the store-gateway is to <strong>leverage on the existing hash ring</strong> to build a per-tenant <strong>subring</strong>, which is then used both by the querier and store-gateway to know to which store-gateway a block belongs to.</p><h3 id=configuration-1>Configuration</h3><ul><li>Shuffle sharding can be enabled in the <strong>store-gateway configuration.</strong> It supports a <strong>default sharding factor,</strong> which is <strong>overridable on a per-tenant basis</strong> and live reloaded during runtime (using the existing limits config).</li><li>The querier already requires the store-gateway configuration when the blocks sharding is enabled. Similarly, when shuffle sharding is enabled the querier will require the store-gateway shuffle sharding configuration as well.</li></ul><h3 id=implementation-notes-1>Implementation notes</h3><p>When shuffle sharding is enabled:</p><ul><li>The <strong>store-gateway</strong> <code>syncUsersBlocks()</code> will build a tenant’s subring for each tenant found scanning the bucket and will skip any tenant not belonging to its shard.<br>Likewise, ShardingMetadataFilter will first build a <strong>tenant’s subring</strong> and then will use the existing logic to filter out blocks not belonging to store-gateway instance itself. The tenant ID can be read from the block’s meta.json.</li><li>The <strong>querier</strong> <code>blocksStoreReplicationSet.GetClientsFor()</code> will first build a <strong>tenant’s subring</strong> and then will use the existing logic to find out to which store-gateway instance each requested block belongs to.</li></ul><h3 id=evaluated-alternatives-1>Evaluated alternatives</h3><p><em>Given the store-gateways already form a ring and building the shuffle sharding based on the ring (like in the write path) doesn’t introduce extra operational complexity, we haven’t discussed alternatives.</em></p><h2 id=querier-ingesters-shuffle-sharding>Querier→ Ingesters shuffle sharding</h2><p>We’re currently discussing/evaluating different options.</p><h3 id=problem>Problem</h3><p>Cortex must guarantee query correctness; transiently incorrect results may be cached and returned forever. The main problem to solve when introducing ingesters shuffle sharding on the read path is to make sure that a querier fetch data from all ingesters having at least 1 sample for a given tenant.</p><p>The problem to solve is: how can a querier efficiently find which ingesters have data for a given tenant? Each option must consider the changing of the set of ingesters and the changing of each tenant’s subring size.</p><h3 id=proposal-use-only-the-information-contained-in-the-ring>Proposal: use only the information contained in the ring.</h3><p><em>This section describes an alternative approach. Discussion is still on-going.</em></p><p>The idea is for the queries to be able to deduce what ingesters could possibly hold data for a given tenant by just consulting the ring (and the per-tenant sub ring sizes). We posit that this is possible with only a single piece of extra information: a single timestamp per ingester saying when the ingester first joined the ring.</p><h4 id=scenario-ingester-scale-up>Scenario: ingester scale up</h4><p>When a new ingester is added to the ring, there will be a set of user subrings that see a change: an ingester being removed, and a new one being added. We need to guarantee that for some time period (the block flush interval), the ingester removed is also consulted for queries.</p><p>To do this, during the subring selection if we encounters an ingester added within the time period, we will add this to the subring but continue node selection as before - in effect, selecting an extra ingester:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-go data-lang=go><span style=display:flex><span><span style=color:#204a87;font-weight:700>var</span> <span style=color:#000;font-weight:700>(</span>
diff --git a/docs/proposals/support-metadata-api/index.html b/docs/proposals/support-metadata-api/index.html
index 995386e6e7..43789f15e7 100644
--- a/docs/proposals/support-metadata-api/index.html
+++ b/docs/proposals/support-metadata-api/index.html
@@ -11,7 +11,7 @@
 At the moment of writing, Cortex does not support the api/v1/metadata endpoint that Prometheus implements as metadata was never propagated via remote write. Recent work is done in Prometheus enables the propagation of metadata."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span class=td-sidebar-nav-active-item>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/receiving_metadata.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Support%20metadata%20API" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#problem-statement>Problem Statement</a></li><li><a href=#potential-solutions>Potential Solutions</a><ul><li><a href=#write-path>Write Path</a></li><li><a href=#read-path>Read Path</a></li></ul></li><li><a href=#conclusions>Conclusions</a></li><li><a href=#references>References</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/support-metadata-api/ aria-disabled=true class="btn-link disabled">Support metadata API</a></li></ol></nav><div class=td-content><h1>Support metadata API</h1><header class=article-meta></header><ul><li>Author: @gotjosh</li><li>Reviewers: @gouthamve, @pracucci</li><li>Date: March 2020</li><li>Status: Accepted</li></ul><h2 id=problem-statement>Problem Statement</h2><p>Prometheus holds metric metadata alongside the contents of a scrape. This metadata (<code>HELP</code>, <code>TYPE</code>, <code>UNIT</code> and <code>METRIC_NAME</code>) enables <a href=https://github.com/prometheus/prometheus/issues/6395>some Prometheus API</a> endpoints to output the metadata for integrations (e.g. <a href=https://github.com/grafana/grafana/pull/21124>Grafana</a>) to consume it.</p><p>At the moment of writing, Cortex does not support the <code>api/v1/metadata</code> endpoint that Prometheus implements as metadata was never propagated via remote write. Recent <a href=https://github.com/prometheus/prometheus/pull/6815/files>work is done in Prometheus</a> enables the propagation of metadata.</p><p>With this in place, remote write integrations such as Cortex can now receive this data and implement the API endpoint. This results in Cortex users being able to enjoy a tiny bit more insight on their metrics.</p><h2 id=potential-solutions>Potential Solutions</h2><p>Before we delve into the solutions, let&rsquo;s set a baseline about how the data is received. This applies almost equally for the two.</p><p>Metadata from Prometheus is sent in the same <a href=https://github.com/prometheus/prometheus/blob/master/prompb/remote.proto><code>WriteRequest</code> proto message</a> that the samples use. It is part of a different field (#3 given #2 is already <a href=https://github.com/cortexproject/cortex/blob/master/pkg/ingester/client/cortex.proto#L36>used interally</a>), the data is a set identified by the metric name - that means it is aggregated across targets, and is sent all at once. Implying, Cortex will receive a single <code>WriteRequest</code> containing a set of the metadata for that instance at an specified interval.</p><p>. It is also important to note that this current process is an intermediary step. Eventually, metadata in a request will be sent alongside samples and only for those included. The solutions proposed, take this nuance into account to avoid coupling between the current and future state of Prometheus, and hopefully do something now that also works for the future.</p><p>As a reference, these are some key numbers regarding the size (and send timings) of the data at hand from our clusters at Grafana Labs:</p><ul><li>On average, metadata (a combination of <code>HELP</code>, <code>TYPE</code>, <code>UNIT</code> and <code>METRIC_NAME</code>) is ~55 bytes uncompressed.</li><li>at GL, on an instance with about 2.6M active series, we hold ~1241 unique metrics in total.</li><li>with that, we can assume that on a worst-case scenario the metadata set for that instance is ~68 kilobytes uncompressed.</li><li>by default, this data is only propagated once every minute (aligning with the default scrape interval), but this can be adjusted.</li><li>Finally, what this gives us is a baseline worst-case scenario formula for the data to store per tenant: <code>~68KB * Replication Factor * # of Instances</code>. Keeping in mind that typically, there&rsquo;s a very high overlap of metadata across instances, and we plan to deduplicate in the ingesters.</li></ul><h3 id=write-path>Write Path</h3><ol><li>Store the metadata directly from the distributors into a cache (e.g. Memcached)</li></ol><p>Since metadata is received all at once, we could directly store into an external cache using the tenant ID as a key, and still, avoid a read-modify-write. However, a very common use case of Cortex is to have multiple Prometheus sending data for the same tenant ID. This complicates things, as it adds a need to have an intermediary merging phase and thus making a read-modify-write inevitable.</p><ol start=2><li>Keep metadata in memory within the ingesters</li></ol><p>Similarly to what we do with sample data, we can keep the metadata in-memory in the ingesters and apply similar semantics. I propose to use the tenant ID as a hash key, distribute it to the ingesters (taking into account the replication factor), using a hash map to keep a set of the metadata across all instances for a single tenant, and implement a configurable time-based purge process to deal with metadata churn. Given, we need to ensure fair-use we also propose implementing limits for both the number of metadata entries we can receive and the size of a single entry.</p><h3 id=read-path>Read Path</h3><p>In my eyes, the read path seems to only have one option. At the moment of writing, Cortex uses a <a href=https://github.com/cortexproject/cortex/blob/master/pkg/querier/dummy.go#L11-L20><code>DummyTargetRetriever</code></a> as a way to signal that these API endpoints are not implemented. We&rsquo;d need to modify the Prometheus interface to support a <code>Context</code> and extract the tenant ID from there. Then, use the tenant ID to query the ingesters for the data, deduplicate it and serve it.</p><h2 id=conclusions>Conclusions</h2><p>I conclude that solution #2 is ideal for this work on the write path. It allows us to use similar semantics to samples, thus reducing operational complexity, and lays a groundwork for when we start receiving metadata alongside samples.</p><p>There&rsquo;s one last piece to address: Allowing metadata to survive rolling restarts. Option #1 handles this well, given the aim would be to use an external cache such as Memcached. Option #2 lacks this, as it does not include any plans to persist this data. Given Prometheus (by default) sends metadata every minute, and we don&rsquo;t need a high level of consistency. We expect that an eventual consistency of up to 1 minute on the default case is deemed acceptable.</p><h2 id=references>References</h2><ul><li><a href=https://docs.google.com/document/d/1LoCWPAIIbGSq59NG3ZYyvkeNb8Ymz28PUKbg_yhAzvE/edit#>Prometheus Propagate metadata via Remote Write Design Doc</a></li><li><a href=https://github.com/prometheus/prometheus/issues/6395>Prometheus Propagate metadata via Remote Write Design Issue</a></li></ul></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/proposals/tenant-deletion/index.html b/docs/proposals/tenant-deletion/index.html
index e4ed5fd52e..4dcbcb1338 100644
--- a/docs/proposals/tenant-deletion/index.html
+++ b/docs/proposals/tenant-deletion/index.html
@@ -14,7 +14,7 @@
 object store for long-term storage of blocks, ingesters disks for short-term storage. Ingesters eventually upload data to long-term storage, various caches: query-frontend, chunks, index and metadata, object store for rules (separate from blocks), object store for alert manager configuration, state for alertmanager instances (notifications and silences)."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span class=td-sidebar-nav-active-item>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/tenant-deletion.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Deletion%20of%20Tenant%20Data%20from%20Blocks%20Storage" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#deletion-of-tenant-data>Deletion of tenant data</a></li><li><a href=#problem>Problem</a></li><li><a href=#background>Background</a></li><li><a href=#proposal>Proposal</a><ul><li><a href=#api-endpoints>API Endpoints</a></li><li><a href=#implementation-asynchronous>Implementation (asynchronous)</a></li><li><a href=#blocks-deletion-marker>Blocks deletion marker</a></li></ul></li><li><a href=#alternatives-considered>Alternatives Considered</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/tenant-deletion/ aria-disabled=true class="btn-link disabled">Deletion of Tenant Data from Blocks Storage</a></li></ol></nav><div class=td-content><h1>Deletion of Tenant Data from Blocks Storage</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/pstibrany>Peter Stibrany</a></li><li>Date: November 2020</li><li>Status: Accepted</li></ul><h2 id=deletion-of-tenant-data>Deletion of tenant data</h2><h2 id=problem>Problem</h2><p>When a tenant is deleted from the external system that controls access to Cortex, we want to clean up tenants data from Cortex as well.</p><h2 id=background>Background</h2><p>When using blocks storage, Cortex stores tenant’s data in several places:</p><ul><li>object store for long-term storage of blocks,</li><li>ingesters disks for short-term storage. Ingesters eventually upload data to long-term storage,</li><li>various caches: query-frontend, chunks, index and metadata,</li><li>object store for rules (separate from blocks),</li><li>object store for alert manager configuration,</li><li>state for alertmanager instances (notifications and silences).</li></ul><p>This document expects that there is an external authorization system in place.
diff --git a/docs/proposals/tenant-retention/index.html b/docs/proposals/tenant-retention/index.html
index 648a751650..bc6fad9916 100644
--- a/docs/proposals/tenant-retention/index.html
+++ b/docs/proposals/tenant-retention/index.html
@@ -11,7 +11,7 @@
 Not every backend storage support bucket policies Bucket policies don&rsquo;t easily allow a per-tenant custom retention Background tenants When using blocks storage, Cortex stores tenant’s data in object store for long-term storage of blocks, tenant id as part of the object store path."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span class=td-sidebar-nav-active-item>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/tenant_retention.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Retention%20of%20Tenant%20Data%20from%20Blocks%20Storage" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#retention-of-tenant-data>Retention of tenant data</a></li><li><a href=#problem>Problem</a></li><li><a href=#background>Background</a><ul><li><a href=#tenants>tenants</a></li><li><a href=#runtime-config>runtime config</a></li></ul></li><li><a href=#proposal>Proposal</a><ul><li><a href=#retention-period-field>retention period field</a></li><li><a href=#implementation>Implementation</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/tenant-retention/ aria-disabled=true class="btn-link disabled">Retention of Tenant Data from Blocks Storage</a></li></ol></nav><div class=td-content><h1>Retention of Tenant Data from Blocks Storage</h1><header class=article-meta></header><ul><li>Author: <a href=https://github.com/Allenzhli>Allenzhli</a></li><li>Date: January 2021</li><li>Status: Accepted, Implemented in <a href=https://github.com/cortexproject/cortex/pull/3879>PR #3879</a>.</li></ul><h2 id=retention-of-tenant-data>Retention of tenant data</h2><h2 id=problem>Problem</h2><p>Metric data is growing over time per-tenant, at the same time, the value of data decreases. We want to have a retention policy like prometheus does. In Cortex, data retention is typically achieved via a bucket policy. However, this has two main issues:</p><ol><li>Not every backend storage support bucket policies</li><li>Bucket policies don&rsquo;t easily allow a per-tenant custom retention</li></ol><h2 id=background>Background</h2><h3 id=tenants>tenants</h3><p>When using blocks storage, Cortex stores tenant’s data in object store for long-term storage of blocks, tenant id as part of the object store path. We discover all tenants via scan the root dir of bucket.</p><h3 id=runtime-config>runtime config</h3><p>Using the &ldquo;overrides&rdquo; mechanism (part of runtime config) already allows for per-tenant settings. See <a href=https://cortexmetrics.io/docs/configuration/arguments/#runtime-configuration-file>runtime-configuration-file</a> for more details. Using it for tenant retention would fit nicely. Admin could set per-tenant retention here, and also have a single global value for tenants that don&rsquo;t have custom value set.</p><h2 id=proposal>Proposal</h2><h3 id=retention-period-field>retention period field</h3><p>We propose to introduce just one new field <code>RetentionPeriod</code> in the Limits struct(defined at pkg/util/validation/limits.go).</p><p><code>RetentionPeriod</code> setting how long historical metric data retention period per-tenant. <code>0</code> is disable.</p><p>Runtime config is reloaded periodically (defaults to 10 seconds), so we can update the retention settings on-the-fly.</p><p>For each tenant, if a tenant-specific <em>runtime_config</em> value exists, it will be used directly, otherwise, if a default <em>limits_config</em> value exists, then the default value will be used; If neither exists, do nothing.</p><h3 id=implementation>Implementation</h3><p>A BlocksCleaner within the Compactor run periodically (which defaults to 15 minutes) and the retention logic will insert into it. The logic should compare retention value to block <code>maxTime</code> and blocks that match <code>maxTime &lt; now - retention</code> will be marked for delete.</p><p>Blocks deletion is not immediate, but follows a two steps process. See <a href=https://cortexmetrics.io/docs/blocks-storage/compactor/#soft-and-hard-blocks-deletion>soft-and-hard-blocks-deletion</a></p></div></main></div></div><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
diff --git a/docs/proposals/timeseries-partitioning-in-compactor/index.html b/docs/proposals/timeseries-partitioning-in-compactor/index.html
index d7639c0057..28112b15db 100644
--- a/docs/proposals/timeseries-partitioning-in-compactor/index.html
+++ b/docs/proposals/timeseries-partitioning-in-compactor/index.html
@@ -12,7 +12,7 @@
 Problem and Requirements Cortex introduced horizontally scaling compactor which allows multiple compactors to compact blocks for a single tenant, sharded by time interval."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span class=td-sidebar-nav-active-item>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/proposals/timeseries-partitioning-in-compactor.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Timeseries%20Partitioning%20in%20Compactor" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#timeseries-partitioning-in-compactor>Timeseries Partitioning in Compactor</a></li><li><a href=#introduction>Introduction</a></li><li><a href=#problem-and-requirements>Problem and Requirements</a></li><li><a href=#design>Design</a><ul><li><a href=#partitioning-strategy>Partitioning strategy</a></li><li><a href=#determining-overlapping-blocks>Determining overlapping blocks</a></li><li><a href=#planning-the-compaction>Planning the compaction</a></li><li><a href=#compaction>Compaction</a></li><li><a href=#compaction-workflow>Compaction Workflow</a></li><li><a href=#clean-up-workflow>Clean up Workflow</a></li></ul></li><li><a href=#performance>Performance</a></li><li><a href=#alternatives-considered>Alternatives Considered</a><ul><li><a href=#dynamic-number-of-partition>Dynamic Number of Partition</a></li><li><a href=#consistent-hashing>Consistent Hashing</a></li><li><a href=#partition-by-metric-name>Partition by metric name</a></li></ul></li><li><a href=#architecture>Architecture</a><ul><li><a href=#planning>Planning</a></li><li><a href=#clean-up>Clean up</a></li></ul></li><li><a href=#changes-required-in-dependencies>Changes Required in Dependencies</a><ul><li><a href=#partitioning-during-compaction-time>Partitioning during compaction time</a></li><li><a href=#source-block-checking>Source block checking</a></li></ul></li><li><a href=#work-plan>Work Plan</a></li><li><a href=#appendix>Appendix</a><ul><li><a href=#risks>Risks</a></li><li><a href=#frequently-asked-questions>Frequently Asked Questions</a></li><li><a href=#compaction-partitioning-examples>Compaction Partitioning Examples</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/proposals/>Proposals</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/proposals/timeseries-partitioning-in-compactor/ aria-disabled=true class="btn-link disabled">Timeseries Partitioning in Compactor</a></li></ol></nav><div class=td-content><h1>Timeseries Partitioning in Compactor</h1><header class=article-meta></header><ul><li>Author: @roystchiang</li><li>Reviewers:</li><li>Date: August 2022</li><li>Status: Proposed</li></ul><h2 id=timeseries-partitioning-in-compactor>Timeseries Partitioning in Compactor</h2><h2 id=introduction>Introduction</h2><p>The compactor is a crucial component in Cortex responsible for deduplication of replicated data, and merging blocks across multiple time intervals together. This proposal will not go into great depth with why the compactor is necessary, but aims to focus on how to scale the compactor as a tenant grows within a Cortex cluster.</p><h2 id=problem-and-requirements>Problem and Requirements</h2><p>Cortex introduced horizontally scaling compactor which allows multiple compactors to compact blocks for a single tenant, sharded by time interval. The compactor is capable of compacting multiple smaller blocks into a larger block, to reduce the duplicated information in index. The following is an illustration of how the shuffle sharding compactor works, where each arrow represents a single compaction that can be carried out independently.
diff --git a/docs/roadmap/index.html b/docs/roadmap/index.html
index b5c4b43ddb..5a0313bcb0 100644
--- a/docs/roadmap/index.html
+++ b/docs/roadmap/index.html
@@ -5,7 +5,7 @@
 Helm charts and other packaging We have a helm chart but it needs work before it can be effectively utilised by different backends. We also don&rsquo;t provide an official set of dashboards and alerts to our users yet."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class="nav-link active" href=/docs/><span class=active>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10>
 <button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-section-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsgetting-started-li><a href=/docs/getting-started/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsgetting-started><span>Getting Started</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsarchitecture-li><a href=/docs/architecture/ title="Cortex Architecture" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsarchitecture><span>Architecture</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsblocks-storage-li><a href=/docs/blocks-storage/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsblocks-storage><span>Blocks Storage</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagequerier-li><a href=/docs/blocks-storage/querier/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagequerier><span>Querier</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagestore-gateway-li><a href=/docs/blocks-storage/store-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagestore-gateway><span>Store-gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagecompactor-li><a href=/docs/blocks-storage/compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagecompactor><span>Compactor</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageproduction-tips-li><a href=/docs/blocks-storage/production-tips/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageproduction-tips><span>Production tips</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebinary-index-header-li><a href=/docs/blocks-storage/binary-index-header/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebinary-index-header><span>Binary index-header</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagebucket-index-li><a href=/docs/blocks-storage/bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagebucket-index><span>Bucket Index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks-li><a href=/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-cortex-cluster-from-chunks-to-blocks><span>Migrate Cortex cluster from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks-li><a href=/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storageconvert-long-term-storage-from-chunks-to-blocks><span>Convert long-term storage from chunks to blocks</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus-li><a href=/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagemigrate-storage-from-thanos-and-prometheus><span>Migrate the storage from Thanos and Prometheus</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsblocks-storagelearn-more-li><a href=/docs/blocks-storage/learn-more/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsblocks-storagelearn-more><span>Learn more</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsconfiguration-li><a href=/docs/configuration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsconfiguration><span>Configuration</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationconfiguration-file-li><a href=/docs/configuration/configuration-file/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationconfiguration-file><span>Configuration file</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationarguments-li><a href=/docs/configuration/arguments/ title="Cortex Arguments" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationarguments><span>Cortex Arguments Explained</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationprometheus-frontend-li><a href=/docs/configuration/prometheus-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationprometheus-frontend><span>Prometheus Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsconfigurationv1guarantees-li><a href=/docs/configuration/v1guarantees/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsconfigurationv1guarantees><span>v1.x Guarantees</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsguides-li><a href=/docs/guides/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsguides><span>Guides</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesrunning-cortex-on-kubernetes-li><a href=/docs/guides/running-cortex-on-kubernetes/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesrunning-cortex-on-kubernetes><span>Running Cortex on Kubernetes</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesgetting-started-with-gossiped-ring-li><a href=/docs/guides/getting-started-with-gossiped-ring/ title="Getting started with gossiped ring" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesgetting-started-with-gossiped-ring><span>Getting started with a gossip ring cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesalertmanager-configuration-li><a href=/docs/guides/alertmanager-configuration/ title="Configuring Notification using Cortex Alertmanager" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesalertmanager-configuration><span>Alertmanager configuration</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesauth-li><a href=/docs/guides/auth/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesauth><span>Authentication and Authorisation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidescapacity-planning-li><a href=/docs/guides/capacity-planning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidescapacity-planning><span>Capacity Planning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesruler-sharding-li><a href=/docs/guides/ruler-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesruler-sharding><span>Config for horizontally scaling the Ruler</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesha-pair-handling-li><a href=/docs/guides/ha-pair-handling/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesha-pair-handling><span>Config for sending HA Pairs data to Cortex</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesencryption-at-rest-li><a href=/docs/guides/encryption-at-rest/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesencryption-at-rest><span>Encryption at Rest</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-rolling-updates-li><a href=/docs/guides/ingesters-rolling-updates/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-rolling-updates><span>Ingesters rolling updates</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesingesters-scaling-up-and-down-li><a href=/docs/guides/ingesters-scaling-up-and-down/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesingesters-scaling-up-and-down><span>Ingesters scaling up and down</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesoverrides-exporter-li><a href=/docs/guides/overrides-exporter/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesoverrides-exporter><span>Overrides Exporter</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestls-li><a href=/docs/guides/tls/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestls><span>Securing communication between Cortex components with TLS</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidessecurity-li><a href=/docs/guides/security/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidessecurity><span>Security</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesshuffle-sharding-li><a href=/docs/guides/shuffle-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesshuffle-sharding><span>Shuffle Sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidestracing-li><a href=/docs/guides/tracing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidestracing><span>Tracing</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideszone-aware-replication-li><a href=/docs/guides/zone-aware-replication/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideszone-aware-replication><span>Zone Aware Replication</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguideslimitations-li><a href=/docs/guides/limitations/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguideslimitations><span>Limitations</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsguidesglossary-li><a href=/docs/guides/glossary/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsguidesglossary><span>Glossary</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docsapi-li><a href=/docs/api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsapi><span>HTTP API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsoperations-li><a href=/docs/operations/ title="Operating Cortex" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsoperations><span>Operations</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-auditor-li><a href=/docs/operations/query-auditor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-auditor><span>Query Auditor (tool)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsquery-tee-li><a href=/docs/operations/query-tee/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsquery-tee><span>Query Tee (service)</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsrequests-mirroring-to-secondary-cluster-li><a href=/docs/operations/requests-mirroring-to-secondary-cluster/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsrequests-mirroring-to-secondary-cluster><span>Requests mirroring to secondary cluster</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsoperationsscaling-query-frontend-li><a href=/docs/operations/scaling-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsoperationsscaling-query-frontend><span>Scaling the Query Frontend</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscase-studies-li><a href=/docs/case-studies/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscase-studies><span>Case Studies</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesgojek-li><a href=/docs/case-studies/gojek/ title="How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesgojek><span>Gojek</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesrewe-digital-li><a href=/docs/case-studies/rewe-digital/ title="How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesrewe-digital><span>REWE digital</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscase-studiesbuoyant-cloud-li><a href=/docs/case-studies/buoyant-cloud/ title="Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus" class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscase-studiesbuoyant-cloud><span>Buoyant Cloud</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docsproposals-li><a href=/docs/proposals/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docsproposals><span>Proposals</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsauth-gateway-li><a href=/docs/proposals/auth-gateway/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsauth-gateway><span>Authentication Gateway</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-bucket-index-li><a href=/docs/proposals/blocks-storage-bucket-index/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-bucket-index><span>Blocks storage bucket index</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblocks-storage-sharding-li><a href=/docs/proposals/blocks-storage-sharding/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblocks-storage-sharding><span>Blocks storage sharding</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalscross-tenant-query-federation-li><a href=/docs/proposals/cross-tenant-query-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalscross-tenant-query-federation><span>Cross-Tenant Query Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-deletion-li><a href=/docs/proposals/tenant-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-deletion><span>Deletion of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsdocumentation-versioning-li><a href=/docs/proposals/documentation-versioning/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsdocumentation-versioning><span>Documentation Versioning</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsgeneralize-modules-li><a href=/docs/proposals/generalize-modules/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsgeneralize-modules><span>Generalize Modules Service to make it extensible</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalshttp-api-design-li><a href=/docs/proposals/http-api-design/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalshttp-api-design><span>HTTP API Design</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsingesters-migration-li><a href=/docs/proposals/ingesters-migration/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsingesters-migration><span>Migrating ingesters from chunks to blocks and back.</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsparallel-compaction-li><a href=/docs/proposals/parallel-compaction/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsparallel-compaction><span>Parallel Compaction by Time Interval</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstenant-retention-li><a href=/docs/proposals/tenant-retention/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstenant-retention><span>Retention of Tenant Data from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsring-multikey-li><a href=/docs/proposals/ring-multikey/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsring-multikey><span>Ring Multikey</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-ha-li><a href=/docs/proposals/ruler-ha/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-ha><span>Ruler HA</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-high-availability-li><a href=/docs/proposals/ruler-high-availability/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-high-availability><span>Ruler High Availability</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsruler-tenant-federation-li><a href=/docs/proposals/ruler-tenant-federation/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsruler-tenant-federation><span>Ruler Tenant Federation</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-alertmanager-li><a href=/docs/proposals/scalable-alertmanager/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-alertmanager><span>Scalable Alertmanager</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsscalable-query-frontend-li><a href=/docs/proposals/scalable-query-frontend/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsscalable-query-frontend><span>Scalable Query Frontend</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-and-zone-awareness-li><a href=/docs/proposals/shuffle-sharding-and-zone-awareness/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-and-zone-awareness><span>Shuffle sharding and zone awareness</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsshuffle-sharding-on-the-read-path-li><a href=/docs/proposals/shuffle-sharding-on-the-read-path/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsshuffle-sharding-on-the-read-path><span>Shuffle sharding on the read path</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalssupport-metadata-api-li><a href=/docs/proposals/support-metadata-api/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalssupport-metadata-api><span>Support metadata API</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalsblock-storage-time-series-deletion-li><a href=/docs/proposals/block-storage-time-series-deletion/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalsblock-storage-time-series-deletion><span>Time Series Deletion from Blocks Storage</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docsproposalstimeseries-partitioning-in-compactor-li><a href=/docs/proposals/timeseries-partitioning-in-compactor/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docsproposalstimeseries-partitioning-in-compactor><span>Timeseries Partitioning in Compactor</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docscontributing-li><a href=/docs/contributing/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docscontributing><span>Contributing</span></a><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinggovernance-li><a href=/docs/contributing/governance/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinggovernance><span>Governance</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributingdesign-patterns-and-code-conventions-li><a href=/docs/contributing/design-patterns-and-code-conventions/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributingdesign-patterns-and-code-conventions><span>Design patterns and Code conventions</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-run-the-website-locally-li><a href=/docs/contributing/how-to-run-the-website-locally/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-run-the-website-locally><span>How to run the website locally</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-upgrade-golang-version-li><a href=/docs/contributing/how-to-upgrade-golang-version/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-upgrade-golang-version><span>How to upgrade Golang version</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-integration-tests-work-li><a href=/docs/contributing/how-integration-tests-work/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-integration-tests-work><span>How integration tests work</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-update-the-build-image-li><a href=/docs/contributing/how-to-update-the-build-image/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-update-the-build-image><span>How to update the build image</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child collapse" id=m-docscontributinghow-to-add-a-maintainer-li><a href=/docs/contributing/how-to-add-a-maintainer/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscontributinghow-to-add-a-maintainer><span>How to add a maintainer</span></a></li></ul></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child active-path" id=m-docsroadmap-li><a href=/docs/roadmap/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__page" id=m-docsroadmap><span class=td-sidebar-nav-active-item>Roadmap</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docschangelog-li><a href=/docs/changelog/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docschangelog><span>Changelog</span></a></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docscode-of-conduct-li><a href=/docs/code-of-conduct/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docscode-of-conduct><span>Code of Conduct</span></a></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/cortexproject/cortex/edit/master/docs/roadmap.md target=_blank><i class="fa fa-edit fa-fw"></i> Edit this page</a>
 <a href="https://github.com/cortexproject/cortex/issues/new?title=Roadmap" target=_blank><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
 <a href=https://github.com/cortexproject/cortex/issues/new target=_blank><i class="fas fa-tasks fa-fw"></i> Create project issue</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#helm-charts-and-other-packaging>Helm charts and other packaging</a></li><li><a href=#auth-gateway>Auth Gateway</a></li><li><a href=#billing-and-usage-analytics>Billing and Usage analytics</a></li><li><a href=#downsampling>Downsampling</a></li><li><a href=#per-metric-retention>Per-metric retention</a></li><li><a href=#exemplar-support>Exemplar support</a></li><li><a href=#scalability>Scalability</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/roadmap/ aria-disabled=true class="btn-link disabled">Roadmap</a></li></ol></nav><div class=td-content><h1>Roadmap</h1><header class=article-meta></header><p>This document highlights some ideas for major features we&rsquo;d like to implement in the near future.
diff --git a/index.html b/index.html
index 76607eaabb..b9944e77a8 100644
--- a/index.html
+++ b/index.html
@@ -22,7 +22,7 @@
           Horizontally scalable, highly available, multi-tenant, long term …"><meta property="og:title" content="Cortex"><meta property="og:description" content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta property="og:type" content="website"><meta property="og:url" content="/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Cortex"><meta itemprop=description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Cortex"><meta name=twitter:description content="A horizontally scalable, highly available, multi-tenant, long term storage for Prometheus."><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-home><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar"><a id=cortex-logo class="navbar-brand d-none" href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/docs/><span>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-default td-outer"><main role=main class=td-main><section id=td-cover-block-0 class="row td-cover-block td-cover-block--height-full js-td-cover td-overlay -bg-white"><div class="container td-overlay__inner my-auto"><div class=row><div class=mx-auto><img src=/images/cortex-horizontal.svg width=300 class=img-fluid></div></div><div class=row><div class=col-12><div class=text-center><h1 class="display-1 mt-0 mt-5 pb-4">Horizontally scalable, highly available, multi-tenant, long term storage for Prometheus.</h1><div class="pt-3 lead"><div class=mx-auto><a class="btn btn-lg btn-primary mr-3 mb-4" href=/docs/>Learn More <i class="fas fa-arrow-alt-circle-right ml-2"></i></a>
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-home><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar"><a id=cortex-logo class="navbar-brand d-none" href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/docs/><span>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-default td-outer"><main role=main class=td-main><section id=td-cover-block-0 class="row td-cover-block td-cover-block--height-full js-td-cover td-overlay -bg-white"><div class="container td-overlay__inner my-auto"><div class=row><div class=mx-auto><img src=/images/cortex-horizontal.svg width=300 class=img-fluid></div></div><div class=row><div class=col-12><div class=text-center><h1 class="display-1 mt-0 mt-5 pb-4">Horizontally scalable, highly available, multi-tenant, long term storage for Prometheus.</h1><div class="pt-3 lead"><div class=mx-auto><a class="btn btn-lg btn-primary mr-3 mb-4" href=/docs/>Learn More <i class="fas fa-arrow-alt-circle-right ml-2"></i></a>
 <a class="btn btn-lg btn-secondary mr-3 mb-4" href=https://github.com/cortexproject/cortex/releases>Releases <i class="fab fa-github ml-2"></i></a></div><p><a class="btn btn-link text-primary" href=#td-block-1 aria-label="Read more"><i class="fa fa-chevron-circle-down" style=font-size:400%></i></a></div></div></div></div></div></section><a id=td-block-1 class=td-offset-anchor></a><section class="row td-box td-box--light position-relative td-box--gradient td-box--height-auto"><div class="container td-arrow-down"><div class="col pb-5"><p class="h2 companies text-center">Companies using Cortex</p></div><div class=adopters-logos><a class=adopter-logo href=https://aspenmesh.io/ rel=nofollow target=_blank><img src=/logos/aspen_mesh.png alt="Aspen Mesh"></a>
 <a class=adopter-logo href=https://aws.amazon.com/prometheus/ rel=nofollow target=_blank><img src=/logos/AWS_logo_RGB.png alt=AWS></a>
 <a class=adopter-logo href=https://buoyant.io/ rel=nofollow target=_blank><img src=/logos/buoyant.svg alt=Buoyant></a>
diff --git a/index.json b/index.json
index fd65f8fe16..5dde1f527c 100644
--- a/index.json
+++ b/index.json
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.101.0"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/favicons/android-96x96.png sizes=96x96><link rel=icon type=image/png href=/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/favicons/android-192x192.png sizes=192x192><title>Cortex</title><meta name=description content><meta property="og:title" content><meta property="og:description" content><meta property="og:type" content="article"><meta property="og:url" content="/index.json"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content><meta property="article:modified_time" content="2024-12-05T13:05:53-08:00"><meta property="og:site_name" content="Cortex"><meta itemprop=name content><meta itemprop=description content><meta itemprop=dateModified content="2024-12-05T13:05:53-08:00"><meta itemprop=wordCount content="0"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content><meta name=twitter:description content><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.101.0"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/favicons/android-96x96.png sizes=96x96><link rel=icon type=image/png href=/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/favicons/android-192x192.png sizes=192x192><title>Cortex</title><meta name=description content><meta property="og:title" content><meta property="og:description" content><meta property="og:type" content="article"><meta property="og:url" content="/index.json"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content><meta property="article:modified_time" content="2024-12-12T06:21:42+09:00"><meta property="og:site_name" content="Cortex"><meta itemprop=name content><meta itemprop=description content><meta itemprop=dateModified content="2024-12-12T06:21:42+09:00"><meta itemprop=wordCount content="0"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content><meta name=twitter:description content><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/docs/><span>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-default td-outer"><main role=main class=td-main></main><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/docs/><span>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-default td-outer"><main role=main class=td-main></main><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
 For a list of trademarks of The Linux Foundation, please see <a class="text-light alert-link" href=https://www.linuxfoundation.org/trademark-usage>Trademark Usage page</a>.</small></div></footer></div><script src=https://cdn.jsdelivr.net/npm/popper.js@1.16.1/dist/umd/popper.min.js integrity=sha384-9/reFTGAW83EW2RDu2S0VKaIzap3H66lZH81PoYlFhbGU+6BZp6G7niu735Sk7lN crossorigin=anonymous></script>
 <script src=https://cdn.jsdelivr.net/npm/bootstrap@4.6.1/dist/js/bootstrap.min.js integrity="sha512-UR25UO94eTnCVwjbXozyeVd6ZqpaAE9naiEUBK/A+QDbfSTQFhPGj5lOR6d8tsgbBk84Ggb5A3EkjsOgPRPcKA==" crossorigin=anonymous></script>
 <script src=/js/tabpane-persist.js></script>
diff --git a/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json b/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json
deleted file mode 100644
index 81647d4a80..0000000000
--- a/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json
+++ /dev/null
@@ -1 +0,0 @@
-[{"body":" Author: Doğukan Teber Date: March 2023 Status: Proposed Overview If you run Cortex for multiple tenants you need to identify your tenants every time they send metrics or query them. This is needed to ensure that metrics can be ingested and queried separately from each other. For this purpose, the Cortex microservices require you to pass a Header called X-Scope-OrgID. The Cortex k8s manifests suggest deploying an NGINX server inside of each tenant which acts as a reverse proxy. Its sole purpose is proxying the traffic and setting the X-Scope-OrgID header for your tenant.\nProposal We propose to solve this problem by adding a gateway that can be considered the entry point for all requests towards Cortex. This gateway will be responsible for handling multi-tenancy features without complicated proxy configuration.\nThis gateway will be located in another repository, cortexproject/auth-gateway. For the first version, basic authentication will be supported. However, as the project grows, it is planned to support different authentication mechanisms according to the needs of the users.\nWe would like to have a simple configuration format. The following is the proposed configuration:\nserver: address: localhost port: 8080 debug: address: localhost port: 8090 tenants: - authentication: basic username: username1 password: password1 id: orgid - authentication: basic username: username2 password: password2 id: orgid distributor: dns_refresh_interval: 3s url: http://distributor:9009 paths: - /api/v1/push - /api/prom/push http_client_timeout: 20 read_timeout: 5 write_timeout: 5 idle_timeout: 7 frontend: url: http://frontend:9010 read_timeout: 5 write_timeout: 5 idle_timeout: 7 ... debug will be used for endpoints that do not require authentication, ie. /metrics, /ready, /pprof. If no paths are given in the component, such as frontend, the default API paths of that component will be registered. If paths are provided, then the provided API paths will override the default API paths. Additionally, the users will be able to specify custom timeouts for each component similar to NGINX.\nNote that only the distributor and query-frontend are mentioned in the example configuration. However, in addition to the distributor and query-frontend, alertmanager and ruler will be supported. The same configurations can be done for these components as well.\nThe time values that are specified in the above configuration are not tested and are only given as an example. The default values for these fields will be decided later when comprehensive tests are conducted.\nFinally, we will ensure the load is balanced between the targets similar to NGINX in which there is a configurable dns refresh parameter and a round-robin between all the hosts.\n","categories":"","description":"","excerpt":" Author: Doğukan Teber Date: March 2023 Status: Proposed Overview If …","ref":"/docs/proposals/auth-gateway/","tags":"","title":"Authentication Gateway"},{"body":" Author: Marco Pracucci Date: November 2020 Status: draft Introduction Queriers and store-gateways, at startup and periodically while running, need to scan the entire object store bucket to find tenants and blocks for each tenants.\nFor each discovered block, they need to fetch the meta.json file and check for the existence of deletion-mark.json (used to signal a block is “marked for deletion”). The meta.json file is immutable and gets cached, but deletion-mark.json can be created at any time and its non-existence can’t be safely cached for long (actually our cache is pretty ineffective regarding this).\nThe number of blocks in the storage linearly increases with the number of tenants, and so does the number of bucket API calls required to scan the bucket.\nFor example, assuming 1 block / day / tenant and a 400d retention, we would have 400 blocks for a single-tenant cluster, but 4M blocks for a 10K tenants cluster (regardless of the number of active series or their QPS).\nWe’re currently testing a cluster running with 6K tenants and we have faced the following issues:\nThe cold start of queriers and store-gateways (empty caches) take tens of minutes. If we increase the concurrency of bucket operations run to scan the bucket, we hit into object store rate limits and/or the 5xx error rate increases The need to run “list objects” API call for queriers and store-gateways to discover blocks and the check for the existence of deletion-mark.json (which requires a read operation for each block and can’t be cached for a long time) represents a significant % of bucket API calls baseline costs, regardless the tenants QPS (costs we have even if the cluster has 0 queries) Goal The goal of this proposal is:\nThe querier should be up and running without having to scan the bucket at all (zero startup time) The querier should not run any “list objects” operation at anytime The querier should require only 1 “get object” operation to get the entire view of a tenant’s blocks Out of scope We believe the same technique described in this proposal could be applied to optimise the store-gateway too (we’ve built a PoC), however to keep the design doc easier to discuss we suggest to keep the store-gateway out of scope and address it in a follow-up design doc.\nProposal We propose to introduce a per-tenant bucket index. The index is a single JSON file containing two main information: list of all completed (non partial) blocks in the bucket + list of all deletion marks. The bucket index is stored in the bucket within the tenant location (eg. /user-1/bucket-index.json) and is kept updated by the compactor.\nThe querier, at query time, checks whether the bucket index for the tenant has already been loaded in memory. If not, the querier will download it and cache it in memory. Given it’s a small file, we expect the lazy download of the bucket index to not significantly impact first query performances.\nWhile in-memory, a background process will keep it updated at periodic intervals (configurable), so that subsequent queries from the same tenant to the same querier instance will use the cached (and periodically updated) bucket index.\nIf a bucket index is unused for a long time (configurable), eg. because that querier instance is not receiving any query from the tenant, the querier will offload it, stopping to keep it updated at regular intervals. This is particularly useful when shuffle sharding is enabled, because a querier will only run queries for a subset of tenants and tenants can be re-sharded across queries in the event of a scale up/down or rolling updates.\nThe bucket index will be also cached on memcached for a short period of time, to reduce the number of “get object” operations in case multiple queriers fetch it in a short period of time (eg. 5 minutes).\nBucket index structure The bucket-index.json is a single file containing the following information:\nVersion List of blocks ID, MinTime, MaxTime, UploadedAt SegmentsFormat (eg. “1-based-6-digits”), SegmentsNum (these two information will be used by the store-gateway) List of block deletion marks ID, DeletionTime Timestamp of when the index has been updated The bucket index intentionally stores a subset of data for each block’s meta.json, in order to keep the size of the index small. The information in the index is enough to run the querier without having to load the full meta.json of each block.\nSize: the expected size of the index is about 150 bytes per block. In the case of a tenant with 400 blocks, its bucket index would be 58KB. The size could be further reduced compressing it: experiments show a compression ratio of about 4x using gzip.\nWhy the compactor writes the bucket index There are few reasons why the compactor may be a good candidate to write the bucket index:\nThe compactor already scans the bucket to have a consistent view of the blocks before running a compaction. Writing the bucket index in the compactor would require no additional bucket API calls. Queriers and store-gateways currently read only from the bucket (no writes). We believe that it may be a nice property to preserve. How to reduce bucket API calls required to discover deletion marks The deletion-mark.json files are stored within each block location. This means that the compactor would still need to run a “get object” operation for every single block in order to find out which block has it, while updating the bucket index.\nTo reduce the number of operations required, we propose to store block deletion marks for all blocks in a per-tenant location (markers/). Discovering all blocks marked for deletion for a given tenant would only require a single “list objects” operation on the /\u003ctenant-id\u003e/markers/ prefix. New markers introduced in the future (eg. tenant deletion mark) could be stored in the same location in order to discover all markers with a single “list objects” operation.\nFor example:\n/user-1/markers/01ER1ZSYHF1FT6RBD8HTVQWX13-deletion-mark.json /user-1/markers/01ER1ZGSX1Q4B41MK1QQ7RHD33-deletion-mark.json /user-1/markers/01ER1ZGWKVFT60YMXT8D3XJMDB-deletion-mark.json What if a bucket index gets stale Queriers and store-gateways don’t need a strongly consistent view over the bucket. Even today, given queriers and store-gateways run a periodic scan of the bucket, their view could be stale up to the “scan interval” (defaults to 15m).\nThe maximum delay between a change into the bucket is picked up by queriers and store-gateways depends on the configuration and is the minimum time between:\nNew blocks uploaded by ingester: min(-querier.query-ingesters-within, -blocks-storage.tsdb.retention-period) (default: 6h) New blocks uploaded/deleted by compactor: -compactor.deletion-delay (default: 12h) In order to guarantee consistent query results, we propose to configure the bucket index max stale period in the querier and fail queries if, because of any issue, the bucket index UpdatedAt age exceeds it.\nThe staleness of each bucket index will be tracked via metrics, in order to be alert on it when it’s close to expiration (but before it happens).\nObject store eventual consistency Object stores like S3 or GCS do guarantee read-after-create consistency but not read-after-update.\nHowever, given the analysis done in “What if a bucket index gets stale” we expect this not to be a real issue, considering we do expect object store reads to not be out of sync for hours.\n","categories":"","description":"","excerpt":" Author: Marco Pracucci Date: November 2020 Status: draft Introduction …","ref":"/docs/proposals/blocks-storage-bucket-index/","tags":"","title":"Blocks storage bucket index"},{"body":" Author: Marco Pracucci Date: March 2020 Status: accepted Problem In Cortex, when using the experimental blocks storage, each querier internally runs the Thanos BucketStore. This means that each querier has a full view over all blocks in the long-term storage and all blocks index headers are loaded in each querier memory. The querier memory usage linearly increase with number and size of all blocks in the storage, imposing a scalability limit to the blocks storage.\nIn this proposal we want to solve this. In particular, we want to:\nShard blocks (index headers) across a pool of nodes Do not compromise HA on the read path (if a node fails, queries should continue to work) Do not compromise correctness (either the query result is correct or it fails) Proposed solution The idea is to introduce a new Cortex service - store-gateway - internally running the Thanos BucketStore. At query time, a querier will run a query fetching the matching series both from ingesters and the subset of gateways holding the related blocks (based on the query time range). Blocks are replicated across the gateways in order to guarantee query results consistency and HA even in the case of a gateway instance failure.\nRing-based sharding and replication In order to build blocks sharding and replication, the store-gateway instances form a ring. Each gateway instance uses a custom MetaFetcherFilter to filter blocks loaded on the instance itself, keeping only blocks whose hash(block-id) is within the tokens range assigned to the gateway instance within the ring.\nWithin a gateway, the blocks synchronization is triggered in two cases:\nPeriodically\nto discover new blocks uploaded by ingesters or compactor, and delete old blocks removed due to retention or by the compactor On-demand when the ring topology changes (the tokens ranges assigned to the gateway instance have changed) It’s important to outline that the sync takes time (typically will have to re-scan the bucket and download new blocks index headers) and Cortex needs to guarantee query results consistency at any given time (see below).\nQuery execution When a querier executes a query, it will need to fetch series both from ingesters and the store-gateway instances.\nFor a given query, the number of blocks to query is expected to be low, especially if the Cortex cluster is running the query-frontend with a 24h query split interval. In this scenario, whatever is the client’s query time range, the query-frontend will split the client’s query into partitioned queries each with up to 24h time range and the querier will likely hit not more than 1 block per partitioned query (except for the last 24h for which blocks may have not been compacted yet).\nGiven this assumption, we want to avoid sending every query to every store-gateway instance. The querier should be able to take an informed decision about the minimum subset of store-gateway instances which needs to query given a time range.\nThe idea is to run the MetaFetcher also within the querier, but without any sharding filter (contrary to the store-gateway). At any given point in time, the querier knows the entire list of blocks in the storage. When the querier executes the Select() (or SelectSorted()) it does:\nCompute the list of blocks by the query time range Compute the minimum list of store-gateway instances containing the required blocks (using the information from the ring) Fetch series from ingesters and the matching store-gateway instances Merge and deduplicate received series Optimization: can be skipped if the querier hits only 1 store-gateway Query results consistency When a querier executes a query, it should guarantee that either all blocks matching the time range are queried or the query fails.\nHowever, due to the (intentional) lack of a strong coordination between queriers and store-gateways, and the ring topology which can change any time, there’s no guarantee that the blocks assigned to a store-gateway shard are effectively loaded on the store-gateway itself at any given point in time.\nThe idea is introduce a consistency check in the querier. When a store-gateway receives a request from the querier, the store-gateway includes in the response the list of block IDs currently loaded on the store-gateway itself. The querier can then merge the list of block IDs received from all store-gateway hit, and match it against the list of block IDs computed at the beginning of the query execution.\nThere are three possible scenarios:\nThe list match: all good All the blocks known by the querier are within the list of blocks returned by store-gateway, but the store-gateway also included blocks unknown to the querier: all good (it means the store-gateways have discovered and loaded new blocks before the querier discovered them) Some blocks known by the querier are not within the list of blocks returned by store-gateway: potential consistency issue We want to protect from a partial results response which may occur in the case #3. However, there are some legit cases which, if not handled, would lead to frequent false positives. Given the querier and store-gateway instances independently scan the bucket at a regular interval (to find new blocks or deleted blocks), we may be in one of the following cases:\na. The querier has discovered new blocks before the store-gateway successfully discovered and loaded them b. The store-gateway has offloaded blocks “marked for deletion” before the querier\nTo protect from case (a), we can exclude the blocks which have been uploaded in the last X time from the consistency check (same technique already used in other Thanos components). This X delay time is used to give the store-gateway enough time to discover and load new blocks, before the querier consider them for the consistency check. This value X should be greater than the -experimental.blocks-storage.bucket-store.consistency-delay, because we do expect the querier to consider a block for consistency check once it’s reasonably safe to assume that its store-gateway already loaded it.\nTo protect from case (b) we need to understand how blocks are offloaded. The BucketStore (running within the store-gateway) offloads a block as soon as it’s not returned by the MetaFetcher. This means we can configure the MetaFetcher with a IgnoreDeletionMarkFilter with a delay of X (could be the same value used for case (a)) and in the querier exclude the blocks which have been marked for deletion more than X time ago from the consistency check.\nTrade-offs The proposed solution comes with the following trade-offs:\nA querier is not ready until it has completed an initial full scan of the bucket, downloading the meta.json file of every block A store-gateway is not ready until it has completed an initial full scan of the bucket, downloading the meta.json and index header of each block matching its shard If a querier hits 2+ store-gateways it may receive duplicated series if the 2+ store-gateways share some blocks due to the replication factor ","categories":"","description":"","excerpt":" Author: Marco Pracucci Date: March 2020 Status: accepted Problem In …","ref":"/docs/proposals/blocks-storage-sharding/","tags":"","title":"Blocks storage sharding"},{"body":" Cortex can be configured using a YAML file - specified using the -config.file flag - or CLI flags. In case you combine both, CLI flags take precedence over the YAML config file.\nThe current configuration of any Cortex component can be seen by visiting the /config HTTP path. Passwords are filtered out of this endpoint.\nReference To specify which configuration file to load, pass the -config.file flag at the command line. The file is written in YAML format, defined by the scheme below. Brackets indicate that a parameter is optional.\nGeneric placeholders \u003cboolean\u003e: a boolean that can take the values true or false \u003cint\u003e: any integer matching the regular expression [1-9]+[0-9]* \u003cduration\u003e: a duration matching the regular expression [0-9]+(ns|us|µs|ms|s|m|h|d|w|y) where y = 365 days. \u003cstring\u003e: a regular string \u003curl\u003e: a URL \u003cprefix\u003e: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used) \u003crelabel_config\u003e: a Prometheus relabeling configuration. \u003ctime\u003e: a timestamp, with available formats: 2006-01-20 (midnight, local timezone), 2006-01-20T15:04 (local timezone), and RFC 3339 formats: 2006-01-20T15:04:05Z (UTC) or 2006-01-20T15:04:05+07:00 (explicit timezone) Use environment variables in the configuration You can use environment variable references in the config file to set values that need to be configurable during deployment by using the -config.expand-env flag. To do this, use:\n${VAR} Where VAR is the name of the environment variable.\nEach variable reference is replaced at startup by the value of the environment variable. The replacement is case-sensitive and occurs before the YAML file is parsed. References to undefined variables are replaced by empty strings unless you specify a default value or custom error text.\nTo specify a default value, use:\n${VAR:default_value} Where default_value is the value to use if the environment variable is undefined.\nSupported contents and default values of the config file # Comma-separated list of Cortex modules to load. The alias 'all' can be used in # the list to load a number of core modules and will enable single-binary mode. # Use '-modules' command line flag to get a list of available modules, and to # see which modules are included in 'all'. # CLI flag: -target [target: \u003cstring\u003e | default = \"all\"] # Set to false to disable auth. # CLI flag: -auth.enabled [auth_enabled: \u003cboolean\u003e | default = true] # HTTP path prefix for Cortex API. # CLI flag: -http.prefix [http_prefix: \u003cstring\u003e | default = \"/api/prom\"] api: # Use GZIP compression for API responses. Some endpoints serve large YAML or # JSON blobs which can benefit from compression. # CLI flag: -api.response-compression-enabled [response_compression_enabled: \u003cboolean\u003e | default = false] # HTTP URL path under which the Alertmanager ui and api will be served. # CLI flag: -http.alertmanager-http-prefix [alertmanager_http_prefix: \u003cstring\u003e | default = \"/alertmanager\"] # HTTP URL path under which the Prometheus api will be served. # CLI flag: -http.prometheus-http-prefix [prometheus_http_prefix: \u003cstring\u003e | default = \"/prometheus\"] # Which HTTP Request headers to add to logs # CLI flag: -api.http-request-headers-to-log [http_request_headers_to_log: \u003clist of string\u003e | default = []] # Regex for CORS origin. It is fully anchored. Example: # 'https?://(domain1|domain2)\\.com' # CLI flag: -server.cors-origin [cors_origin: \u003cstring\u003e | default = \".*\"] # If enabled, build Info API will be served by query frontend or querier. # CLI flag: -api.build-info-enabled [build_info_enabled: \u003cboolean\u003e | default = false] # Choose default codec for querier response serialization. Supports 'json' and # 'protobuf'. # CLI flag: -api.querier-default-codec [querier_default_codec: \u003cstring\u003e | default = \"json\"] # The server_config configures the HTTP and gRPC server of the launched # service(s). [server: \u003cserver_config\u003e] # The distributor_config configures the Cortex distributor. [distributor: \u003cdistributor_config\u003e] # The querier_config configures the Cortex querier. [querier: \u003cquerier_config\u003e] # The ingester_client_config configures how the Cortex distributors connect to # the ingesters. [ingester_client: \u003cingester_client_config\u003e] # The ingester_config configures the Cortex ingester. [ingester: \u003cingester_config\u003e] # The flusher_config configures the WAL flusher target, used to manually run # one-time flushes when scaling down ingesters. [flusher: \u003cflusher_config\u003e] # The storage_config configures the storage type Cortex uses. [storage: \u003cstorage_config\u003e] # The limits_config configures default and per-tenant limits imposed by Cortex # services (ie. distributor, ingester, ...). [limits: \u003climits_config\u003e] # The frontend_worker_config configures the worker - running within the Cortex # querier - picking up and executing queries enqueued by the query-frontend or # query-scheduler. [frontend_worker: \u003cfrontend_worker_config\u003e] # The query_frontend_config configures the Cortex query-frontend. [frontend: \u003cquery_frontend_config\u003e] # The query_range_config configures the query splitting and caching in the # Cortex query-frontend. [query_range: \u003cquery_range_config\u003e] # The blocks_storage_config configures the blocks storage. [blocks_storage: \u003cblocks_storage_config\u003e] # The compactor_config configures the compactor for the blocks storage. [compactor: \u003ccompactor_config\u003e] # The store_gateway_config configures the store-gateway service used by the # blocks storage. [store_gateway: \u003cstore_gateway_config\u003e] tenant_federation: # If enabled on all Cortex services, queries can be federated across multiple # tenants. The tenant IDs involved need to be specified separated by a `|` # character in the `X-Scope-OrgID` header (experimental). # CLI flag: -tenant-federation.enabled [enabled: \u003cboolean\u003e | default = false] # The ruler_config configures the Cortex ruler. [ruler: \u003cruler_config\u003e] # The ruler_storage_config configures the Cortex ruler storage backend. [ruler_storage: \u003cruler_storage_config\u003e] # The configs_config configures the Cortex Configs DB and API. [configs: \u003cconfigs_config\u003e] # The alertmanager_config configures the Cortex alertmanager. [alertmanager: \u003calertmanager_config\u003e] # The alertmanager_storage_config configures the Cortex alertmanager storage # backend. [alertmanager_storage: \u003calertmanager_storage_config\u003e] # The runtime_configuration_storage_config configures the storage backend for # the runtime configuration file. [runtime_config: \u003cruntime_configuration_storage_config\u003e] # The memberlist_config configures the Gossip memberlist. [memberlist: \u003cmemberlist_config\u003e] query_scheduler: # If a querier disconnects without sending notification about graceful # shutdown, the query-scheduler will keep the querier in the tenant's shard # until the forget delay has passed. This feature is useful to reduce the # blast radius when shuffle-sharding is enabled. # CLI flag: -query-scheduler.querier-forget-delay [querier_forget_delay: \u003cduration\u003e | default = 0s] # This configures the gRPC client used to report errors back to the # query-frontend. grpc_client_config: # gRPC client max receive message size (bytes). # CLI flag: -query-scheduler.grpc-client-config.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # gRPC client max send message size (bytes). # CLI flag: -query-scheduler.grpc-client-config.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 16777216] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy', 'snappy-block' ,'zstd' and '' (disable compression) # CLI flag: -query-scheduler.grpc-client-config.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # Rate limit for gRPC client; 0 means disabled. # CLI flag: -query-scheduler.grpc-client-config.grpc-client-rate-limit [rate_limit: \u003cfloat\u003e | default = 0] # Rate limit burst for gRPC client. # CLI flag: -query-scheduler.grpc-client-config.grpc-client-rate-limit-burst [rate_limit_burst: \u003cint\u003e | default = 0] # Enable backoff and retry when we hit ratelimits. # CLI flag: -query-scheduler.grpc-client-config.backoff-on-ratelimits [backoff_on_ratelimits: \u003cboolean\u003e | default = false] backoff_config: # Minimum delay when backing off. # CLI flag: -query-scheduler.grpc-client-config.backoff-min-period [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -query-scheduler.grpc-client-config.backoff-max-period [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -query-scheduler.grpc-client-config.backoff-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS in the GRPC client. This flag needs to be enabled when any # other TLS flag is set. If set to false, insecure connection to gRPC server # will be used. # CLI flag: -query-scheduler.grpc-client-config.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -query-scheduler.grpc-client-config.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -query-scheduler.grpc-client-config.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. # If not set, the host's root CA certificates are used. # CLI flag: -query-scheduler.grpc-client-config.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -query-scheduler.grpc-client-config.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -query-scheduler.grpc-client-config.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # The tracing_config configures backends cortex uses. [tracing: \u003ctracing_config\u003e] alertmanager_config The alertmanager_config configures the Cortex alertmanager.\n# Base path for data storage. # CLI flag: -alertmanager.storage.path [data_dir: \u003cstring\u003e | default = \"data/\"] # How long to keep data for. # CLI flag: -alertmanager.storage.retention [retention: \u003cduration\u003e | default = 120h] # The URL under which Alertmanager is externally reachable (for example, if # Alertmanager is served via a reverse proxy). Used for generating relative and # absolute links back to Alertmanager itself. If the URL has a path portion, it # will be used to prefix all HTTP endpoints served by Alertmanager. If omitted, # relevant URL components will be derived automatically. # CLI flag: -alertmanager.web.external-url [external_url: \u003curl\u003e | default = ] # How frequently to poll Cortex configs # CLI flag: -alertmanager.configs.poll-interval [poll_interval: \u003cduration\u003e | default = 15s] # Maximum size (bytes) of an accepted HTTP request body. # CLI flag: -alertmanager.max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 16777216] # Shard tenants across multiple alertmanager instances. # CLI flag: -alertmanager.sharding-enabled [sharding_enabled: \u003cboolean\u003e | default = false] sharding_ring: # The key-value store used to share the hash ring across multiple instances. kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -alertmanager.sharding-ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -alertmanager.sharding-ring.prefix [prefix: \u003cstring\u003e | default = \"alertmanagers/\"] dynamodb: # Region to access dynamodb. # CLI flag: -alertmanager.sharding-ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -alertmanager.sharding-ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -alertmanager.sharding-ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -alertmanager.sharding-ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -alertmanager.sharding-ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: alertmanager.sharding-ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: alertmanager.sharding-ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -alertmanager.sharding-ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -alertmanager.sharding-ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -alertmanager.sharding-ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -alertmanager.sharding-ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -alertmanager.sharding-ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 15s] # The heartbeat timeout after which alertmanagers are considered unhealthy # within the ring. 0 = never (timeout disabled). # CLI flag: -alertmanager.sharding-ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # The replication factor to use when sharding the alertmanager. # CLI flag: -alertmanager.sharding-ring.replication-factor [replication_factor: \u003cint\u003e | default = 3] # True to enable zone-awareness and replicate alerts across different # availability zones. # CLI flag: -alertmanager.sharding-ring.zone-awareness-enabled [zone_awareness_enabled: \u003cboolean\u003e | default = false] # The sleep seconds when alertmanager is shutting down. Need to be close to or # larger than KV Store information propagation delay # CLI flag: -alertmanager.sharding-ring.final-sleep [final_sleep: \u003cduration\u003e | default = 0s] # Timeout for waiting on alertmanager to become desired state in the ring. # CLI flag: -alertmanager.sharding-ring.wait-instance-state-timeout [wait_instance_state_timeout: \u003cduration\u003e | default = 10m] # Name of network interface to read address from. # CLI flag: -alertmanager.sharding-ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # The availability zone where this instance is running. Required if # zone-awareness is enabled. # CLI flag: -alertmanager.sharding-ring.instance-availability-zone [instance_availability_zone: \u003cstring\u003e | default = \"\"] # Filename of fallback config to use if none specified for instance. # CLI flag: -alertmanager.configs.fallback [fallback_config_file: \u003cstring\u003e | default = \"\"] # Root of URL to generate if config is http://internal.monitor # CLI flag: -alertmanager.configs.auto-webhook-root [auto_webhook_root: \u003cstring\u003e | default = \"\"] cluster: # Listen address and port for the cluster. Not specifying this flag disables # high-availability mode. # CLI flag: -alertmanager.cluster.listen-address [listen_address: \u003cstring\u003e | default = \"0.0.0.0:9094\"] # Explicit address or hostname to advertise in cluster. # CLI flag: -alertmanager.cluster.advertise-address [advertise_address: \u003cstring\u003e | default = \"\"] # Comma-separated list of initial peers. # CLI flag: -alertmanager.cluster.peers [peers: \u003cstring\u003e | default = \"\"] # Time to wait between peers to send notifications. # CLI flag: -alertmanager.cluster.peer-timeout [peer_timeout: \u003cduration\u003e | default = 15s] # The interval between sending gossip messages. By lowering this value (more # frequent) gossip messages are propagated across cluster more quickly at the # expense of increased bandwidth usage. # CLI flag: -alertmanager.cluster.gossip-interval [gossip_interval: \u003cduration\u003e | default = 200ms] # The interval between gossip state syncs. Setting this interval lower (more # frequent) will increase convergence speeds across larger clusters at the # expense of increased bandwidth usage. # CLI flag: -alertmanager.cluster.push-pull-interval [push_pull_interval: \u003cduration\u003e | default = 1m] # Enable the experimental alertmanager config api. # CLI flag: -experimental.alertmanager.enable-api [enable_api: \u003cboolean\u003e | default = false] # Maximum number of concurrent GET API requests before returning an error. # CLI flag: -alertmanager.api-concurrency [api_concurrency: \u003cint\u003e | default = 0] # Alertmanager alerts Garbage collection interval. # CLI flag: -alertmanager.alerts-gc-interval [gc_interval: \u003cduration\u003e | default = 30m] alertmanager_client: # Timeout for downstream alertmanagers. # CLI flag: -alertmanager.alertmanager-client.remote-timeout [remote_timeout: \u003cduration\u003e | default = 2s] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -alertmanager.alertmanager-client.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -alertmanager.alertmanager-client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -alertmanager.alertmanager-client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -alertmanager.alertmanager-client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -alertmanager.alertmanager-client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -alertmanager.alertmanager-client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy' and '' (disable compression) # CLI flag: -alertmanager.alertmanager-client.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # gRPC client max receive message size (bytes). # CLI flag: -alertmanager.alertmanager-client.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 16777216] # gRPC client max send message size (bytes). # CLI flag: -alertmanager.alertmanager-client.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 4194304] # The interval between persisting the current alertmanager state (notification # log and silences) to object storage. This is only used when sharding is # enabled. This state is read when all replicas for a shard can not be # contacted. In this scenario, having persisted the state more frequently will # result in potentially fewer lost silences, and fewer duplicate notifications. # CLI flag: -alertmanager.persist-interval [persist_interval: \u003cduration\u003e | default = 15m] # Comma separated list of tenants whose alerts this alertmanager can process. If # specified, only these tenants will be handled by alertmanager, otherwise this # alertmanager can process alerts from all tenants. # CLI flag: -alertmanager.enabled-tenants [enabled_tenants: \u003cstring\u003e | default = \"\"] # Comma separated list of tenants whose alerts this alertmanager cannot process. # If specified, a alertmanager that would normally pick the specified tenant(s) # for processing will ignore them instead. # CLI flag: -alertmanager.disabled-tenants [disabled_tenants: \u003cstring\u003e | default = \"\"] alertmanager_storage_config The alertmanager_storage_config configures the Cortex alertmanager storage backend.\n# Backend storage to use. Supported backends are: s3, gcs, azure, swift, # filesystem, configdb, local. # CLI flag: -alertmanager-storage.backend [backend: \u003cstring\u003e | default = \"s3\"] s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an # S3-compatible service in hostname:port format. # CLI flag: -alertmanager-storage.s3.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # S3 region. If unset, the client will issue a S3 GetBucketLocation API call # to autodetect it. # CLI flag: -alertmanager-storage.s3.region [region: \u003cstring\u003e | default = \"\"] # S3 bucket name # CLI flag: -alertmanager-storage.s3.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # S3 secret access key # CLI flag: -alertmanager-storage.s3.secret-access-key [secret_access_key: \u003cstring\u003e | default = \"\"] # S3 access key ID # CLI flag: -alertmanager-storage.s3.access-key-id [access_key_id: \u003cstring\u003e | default = \"\"] # If enabled, use http:// for the S3 endpoint instead of https://. This could # be useful in local dev/test environments while using an S3-compatible # backend storage, like Minio. # CLI flag: -alertmanager-storage.s3.insecure [insecure: \u003cboolean\u003e | default = false] # The signature version to use for authenticating against S3. Supported values # are: v4, v2. # CLI flag: -alertmanager-storage.s3.signature-version [signature_version: \u003cstring\u003e | default = \"v4\"] # The s3 bucket lookup style. Supported values are: auto, virtual-hosted, # path. # CLI flag: -alertmanager-storage.s3.bucket-lookup-type [bucket_lookup_type: \u003cstring\u003e | default = \"auto\"] # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum # algorithm to verify the provided digest. If false, use CRC32C algorithm # instead. # CLI flag: -alertmanager-storage.s3.send-content-md5 [send_content_md5: \u003cboolean\u003e | default = true] # The list api version. Supported values are: v1, v2, and ''. # CLI flag: -alertmanager-storage.s3.list-objects-version [list_objects_version: \u003cstring\u003e | default = \"\"] # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: alertmanager-storage [sse: \u003cs3_sse_config\u003e] http: # The time an idle connection will remain idle before closing. # CLI flag: -alertmanager-storage.s3.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -alertmanager-storage.s3.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -alertmanager-storage.s3.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -alertmanager-storage.s3.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -alertmanager-storage.s3.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -alertmanager-storage.s3.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -alertmanager-storage.s3.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -alertmanager-storage.s3.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] gcs: # GCS bucket name # CLI flag: -alertmanager-storage.gcs.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # JSON representing either a Google Developers Console client_credentials.json # file or a Google Developers service account key file. If empty, fallback to # Google default logic. # CLI flag: -alertmanager-storage.gcs.service-account [service_account: \u003cstring\u003e | default = \"\"] azure: # Azure storage account name # CLI flag: -alertmanager-storage.azure.account-name [account_name: \u003cstring\u003e | default = \"\"] # Azure storage account key # CLI flag: -alertmanager-storage.azure.account-key [account_key: \u003cstring\u003e | default = \"\"] # The values of `account-name` and `endpoint-suffix` values will not be # ignored if `connection-string` is set. Use this method over `account-key` if # you need to authenticate via a SAS token or if you use the Azurite emulator. # CLI flag: -alertmanager-storage.azure.connection-string [connection_string: \u003cstring\u003e | default = \"\"] # Azure storage container name # CLI flag: -alertmanager-storage.azure.container-name [container_name: \u003cstring\u003e | default = \"\"] # Azure storage endpoint suffix without schema. The account name will be # prefixed to this value to create the FQDN # CLI flag: -alertmanager-storage.azure.endpoint-suffix [endpoint_suffix: \u003cstring\u003e | default = \"\"] # Number of retries for recoverable errors # CLI flag: -alertmanager-storage.azure.max-retries [max_retries: \u003cint\u003e | default = 20] # Deprecated: Azure storage MSI resource. It will be set automatically by # Azure SDK. # CLI flag: -alertmanager-storage.azure.msi-resource [msi_resource: \u003cstring\u003e | default = \"\"] # Azure storage MSI resource managed identity client Id. If not supplied # default Azure credential will be used. Set it to empty if you need to # authenticate via Azure Workload Identity. # CLI flag: -alertmanager-storage.azure.user-assigned-id [user_assigned_id: \u003cstring\u003e | default = \"\"] http: # The time an idle connection will remain idle before closing. # CLI flag: -alertmanager-storage.azure.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -alertmanager-storage.azure.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -alertmanager-storage.azure.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -alertmanager-storage.azure.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -alertmanager-storage.azure.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -alertmanager-storage.azure.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -alertmanager-storage.azure.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -alertmanager-storage.azure.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] swift: # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -alertmanager-storage.swift.auth-version [auth_version: \u003cint\u003e | default = 0] # OpenStack Swift authentication URL # CLI flag: -alertmanager-storage.swift.auth-url [auth_url: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential ID. # CLI flag: -alertmanager-storage.swift.application-credential-id [application_credential_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential name. # CLI flag: -alertmanager-storage.swift.application-credential-name [application_credential_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential secret. # CLI flag: -alertmanager-storage.swift.application-credential-secret [application_credential_secret: \u003cstring\u003e | default = \"\"] # OpenStack Swift username. # CLI flag: -alertmanager-storage.swift.username [username: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -alertmanager-storage.swift.user-domain-name [user_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -alertmanager-storage.swift.user-domain-id [user_domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user ID. # CLI flag: -alertmanager-storage.swift.user-id [user_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift API key. # CLI flag: -alertmanager-storage.swift.password [password: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -alertmanager-storage.swift.domain-id [domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -alertmanager-storage.swift.domain-name [domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift project ID (v2,v3 auth only). # CLI flag: -alertmanager-storage.swift.project-id [project_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift project name (v2,v3 auth only). # CLI flag: -alertmanager-storage.swift.project-name [project_name: \u003cstring\u003e | default = \"\"] # ID of the OpenStack Swift project's domain (v3 auth only), only needed if it # differs the from user domain. # CLI flag: -alertmanager-storage.swift.project-domain-id [project_domain_id: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift project's domain (v3 auth only), only needed if # it differs from the user domain. # CLI flag: -alertmanager-storage.swift.project-domain-name [project_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift Region to use (v2,v3 auth only). # CLI flag: -alertmanager-storage.swift.region-name [region_name: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift container to put chunks in. # CLI flag: -alertmanager-storage.swift.container-name [container_name: \u003cstring\u003e | default = \"\"] # Max retries on requests error. # CLI flag: -alertmanager-storage.swift.max-retries [max_retries: \u003cint\u003e | default = 3] # Time after which a connection attempt is aborted. # CLI flag: -alertmanager-storage.swift.connect-timeout [connect_timeout: \u003cduration\u003e | default = 10s] # Time after which an idle request is aborted. The timeout watchdog is reset # each time some data is received, so the timeout triggers after X time no # data is received on a request. # CLI flag: -alertmanager-storage.swift.request-timeout [request_timeout: \u003cduration\u003e | default = 5s] filesystem: # Local filesystem storage directory. # CLI flag: -alertmanager-storage.filesystem.dir [dir: \u003cstring\u003e | default = \"\"] # The configstore_config configures the config database storing rules and # alerts, and is used by the Cortex alertmanager. # The CLI flags prefix for this block config is: alertmanager-storage [configdb: \u003cconfigstore_config\u003e] local: # Path at which alertmanager configurations are stored. # CLI flag: -alertmanager-storage.local.path [path: \u003cstring\u003e | default = \"\"] blocks_storage_config The blocks_storage_config configures the blocks storage.\n# Backend storage to use. Supported backends are: s3, gcs, azure, swift, # filesystem. # CLI flag: -blocks-storage.backend [backend: \u003cstring\u003e | default = \"s3\"] s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an # S3-compatible service in hostname:port format. # CLI flag: -blocks-storage.s3.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # S3 region. If unset, the client will issue a S3 GetBucketLocation API call # to autodetect it. # CLI flag: -blocks-storage.s3.region [region: \u003cstring\u003e | default = \"\"] # S3 bucket name # CLI flag: -blocks-storage.s3.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # S3 secret access key # CLI flag: -blocks-storage.s3.secret-access-key [secret_access_key: \u003cstring\u003e | default = \"\"] # S3 access key ID # CLI flag: -blocks-storage.s3.access-key-id [access_key_id: \u003cstring\u003e | default = \"\"] # If enabled, use http:// for the S3 endpoint instead of https://. This could # be useful in local dev/test environments while using an S3-compatible # backend storage, like Minio. # CLI flag: -blocks-storage.s3.insecure [insecure: \u003cboolean\u003e | default = false] # The signature version to use for authenticating against S3. Supported values # are: v4, v2. # CLI flag: -blocks-storage.s3.signature-version [signature_version: \u003cstring\u003e | default = \"v4\"] # The s3 bucket lookup style. Supported values are: auto, virtual-hosted, # path. # CLI flag: -blocks-storage.s3.bucket-lookup-type [bucket_lookup_type: \u003cstring\u003e | default = \"auto\"] # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum # algorithm to verify the provided digest. If false, use CRC32C algorithm # instead. # CLI flag: -blocks-storage.s3.send-content-md5 [send_content_md5: \u003cboolean\u003e | default = true] # The list api version. Supported values are: v1, v2, and ''. # CLI flag: -blocks-storage.s3.list-objects-version [list_objects_version: \u003cstring\u003e | default = \"\"] # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: blocks-storage [sse: \u003cs3_sse_config\u003e] http: # The time an idle connection will remain idle before closing. # CLI flag: -blocks-storage.s3.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -blocks-storage.s3.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -blocks-storage.s3.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -blocks-storage.s3.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -blocks-storage.s3.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -blocks-storage.s3.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -blocks-storage.s3.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -blocks-storage.s3.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] gcs: # GCS bucket name # CLI flag: -blocks-storage.gcs.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # JSON representing either a Google Developers Console client_credentials.json # file or a Google Developers service account key file. If empty, fallback to # Google default logic. # CLI flag: -blocks-storage.gcs.service-account [service_account: \u003cstring\u003e | default = \"\"] azure: # Azure storage account name # CLI flag: -blocks-storage.azure.account-name [account_name: \u003cstring\u003e | default = \"\"] # Azure storage account key # CLI flag: -blocks-storage.azure.account-key [account_key: \u003cstring\u003e | default = \"\"] # The values of `account-name` and `endpoint-suffix` values will not be # ignored if `connection-string` is set. Use this method over `account-key` if # you need to authenticate via a SAS token or if you use the Azurite emulator. # CLI flag: -blocks-storage.azure.connection-string [connection_string: \u003cstring\u003e | default = \"\"] # Azure storage container name # CLI flag: -blocks-storage.azure.container-name [container_name: \u003cstring\u003e | default = \"\"] # Azure storage endpoint suffix without schema. The account name will be # prefixed to this value to create the FQDN # CLI flag: -blocks-storage.azure.endpoint-suffix [endpoint_suffix: \u003cstring\u003e | default = \"\"] # Number of retries for recoverable errors # CLI flag: -blocks-storage.azure.max-retries [max_retries: \u003cint\u003e | default = 20] # Deprecated: Azure storage MSI resource. It will be set automatically by # Azure SDK. # CLI flag: -blocks-storage.azure.msi-resource [msi_resource: \u003cstring\u003e | default = \"\"] # Azure storage MSI resource managed identity client Id. If not supplied # default Azure credential will be used. Set it to empty if you need to # authenticate via Azure Workload Identity. # CLI flag: -blocks-storage.azure.user-assigned-id [user_assigned_id: \u003cstring\u003e | default = \"\"] http: # The time an idle connection will remain idle before closing. # CLI flag: -blocks-storage.azure.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -blocks-storage.azure.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -blocks-storage.azure.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -blocks-storage.azure.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -blocks-storage.azure.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -blocks-storage.azure.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -blocks-storage.azure.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -blocks-storage.azure.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] swift: # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -blocks-storage.swift.auth-version [auth_version: \u003cint\u003e | default = 0] # OpenStack Swift authentication URL # CLI flag: -blocks-storage.swift.auth-url [auth_url: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential ID. # CLI flag: -blocks-storage.swift.application-credential-id [application_credential_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential name. # CLI flag: -blocks-storage.swift.application-credential-name [application_credential_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential secret. # CLI flag: -blocks-storage.swift.application-credential-secret [application_credential_secret: \u003cstring\u003e | default = \"\"] # OpenStack Swift username. # CLI flag: -blocks-storage.swift.username [username: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -blocks-storage.swift.user-domain-name [user_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -blocks-storage.swift.user-domain-id [user_domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user ID. # CLI flag: -blocks-storage.swift.user-id [user_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift API key. # CLI flag: -blocks-storage.swift.password [password: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -blocks-storage.swift.domain-id [domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -blocks-storage.swift.domain-name [domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift project ID (v2,v3 auth only). # CLI flag: -blocks-storage.swift.project-id [project_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift project name (v2,v3 auth only). # CLI flag: -blocks-storage.swift.project-name [project_name: \u003cstring\u003e | default = \"\"] # ID of the OpenStack Swift project's domain (v3 auth only), only needed if it # differs the from user domain. # CLI flag: -blocks-storage.swift.project-domain-id [project_domain_id: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift project's domain (v3 auth only), only needed if # it differs from the user domain. # CLI flag: -blocks-storage.swift.project-domain-name [project_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift Region to use (v2,v3 auth only). # CLI flag: -blocks-storage.swift.region-name [region_name: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift container to put chunks in. # CLI flag: -blocks-storage.swift.container-name [container_name: \u003cstring\u003e | default = \"\"] # Max retries on requests error. # CLI flag: -blocks-storage.swift.max-retries [max_retries: \u003cint\u003e | default = 3] # Time after which a connection attempt is aborted. # CLI flag: -blocks-storage.swift.connect-timeout [connect_timeout: \u003cduration\u003e | default = 10s] # Time after which an idle request is aborted. The timeout watchdog is reset # each time some data is received, so the timeout triggers after X time no # data is received on a request. # CLI flag: -blocks-storage.swift.request-timeout [request_timeout: \u003cduration\u003e | default = 5s] filesystem: # Local filesystem storage directory. # CLI flag: -blocks-storage.filesystem.dir [dir: \u003cstring\u003e | default = \"\"] # This configures how the querier and store-gateway discover and synchronize # blocks stored in the bucket. bucket_store: # Directory to store synchronized TSDB index headers. # CLI flag: -blocks-storage.bucket-store.sync-dir [sync_dir: \u003cstring\u003e | default = \"tsdb-sync\"] # How frequently to scan the bucket, or to refresh the bucket index (if # enabled), in order to look for changes (new blocks shipped by ingesters and # blocks deleted by retention or compaction). # CLI flag: -blocks-storage.bucket-store.sync-interval [sync_interval: \u003cduration\u003e | default = 15m] # Max number of concurrent queries to execute against the long-term storage. # The limit is shared across all tenants. # CLI flag: -blocks-storage.bucket-store.max-concurrent [max_concurrent: \u003cint\u003e | default = 100] # Max number of inflight queries to execute against the long-term storage. The # limit is shared across all tenants. 0 to disable. # CLI flag: -blocks-storage.bucket-store.max-inflight-requests [max_inflight_requests: \u003cint\u003e | default = 0] # Maximum number of concurrent tenants syncing blocks. # CLI flag: -blocks-storage.bucket-store.tenant-sync-concurrency [tenant_sync_concurrency: \u003cint\u003e | default = 10] # Maximum number of concurrent blocks syncing per tenant. # CLI flag: -blocks-storage.bucket-store.block-sync-concurrency [block_sync_concurrency: \u003cint\u003e | default = 20] # Number of Go routines to use when syncing block meta files from object # storage per tenant. # CLI flag: -blocks-storage.bucket-store.meta-sync-concurrency [meta_sync_concurrency: \u003cint\u003e | default = 20] # Minimum age of a block before it's being read. Set it to safe value (e.g # 30m) if your object storage is eventually consistent. GCS and S3 are # (roughly) strongly consistent. # CLI flag: -blocks-storage.bucket-store.consistency-delay [consistency_delay: \u003cduration\u003e | default = 0s] index_cache: # The index cache backend type. Multiple cache backend can be provided as a # comma-separated ordered list to enable the implementation of a cache # hierarchy. Supported values: inmemory, memcached, redis. # CLI flag: -blocks-storage.bucket-store.index-cache.backend [backend: \u003cstring\u003e | default = \"inmemory\"] inmemory: # Maximum size in bytes of in-memory index cache used to speed up blocks # index lookups (shared between all tenants). # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes [max_size_bytes: \u003cint\u003e | default = 1073741824] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items [enabled_items: \u003clist of string\u003e | default = []] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. If # set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should run. # If more keys are specified, internally keys are split into multiple # batches and fetched concurrently, honoring the max concurrency. If set # to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit breaker # becomes half-open. If set to 0, by default open duration is 60 # seconds. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items [enabled_items: \u003clist of string\u003e | default = []] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.index-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. # If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching is # when data is stored in memory instead of fetching data each time. See # https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit breaker # becomes half-open. If set to 0, by default open duration is 60 # seconds. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items [enabled_items: \u003clist of string\u003e | default = []] multilevel: # The maximum number of concurrent asynchronous operations can occur when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of items to backfill per asynchronous operation. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-backfill-items [max_backfill_items: \u003cint\u003e | default = 10000] chunks_cache: # The chunks cache backend type. Single or Multiple cache backend can be # provided. Supported values in single cache: memcached, redis, inmemory, # and '' (disable). Supported values in multi level cache: a comma-separated # list of (inmemory, memcached, redis) # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend [backend: \u003cstring\u003e | default = \"\"] inmemory: # Maximum size in bytes of in-memory chunk cache used to speed up chunk # lookups (shared between all tenants). # CLI flag: -blocks-storage.bucket-store.chunks-cache.inmemory.max-size-bytes [max_size_bytes: \u003cint\u003e | default = 1073741824] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. If # set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should run. # If more keys are specified, internally keys are split into multiple # batches and fetched concurrently, honoring the max concurrency. If set # to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit breaker # becomes half-open. If set to 0, by default open duration is 60 # seconds. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. # If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching is # when data is stored in memory instead of fetching data each time. See # https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit breaker # becomes half-open. If set to 0, by default open duration is 60 # seconds. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] multilevel: # The maximum number of concurrent asynchronous operations can occur when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of items to backfill per asynchronous operation. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-backfill-items [max_backfill_items: \u003cint\u003e | default = 10000] # Size of each subrange that bucket object is split into for better caching. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-size [subrange_size: \u003cint\u003e | default = 16000] # Maximum number of sub-GetRange requests that a single GetRange request can # be split into when fetching chunks. Zero or negative value = unlimited # number of sub-requests. # CLI flag: -blocks-storage.bucket-store.chunks-cache.max-get-range-requests [max_get_range_requests: \u003cint\u003e | default = 3] # TTL for caching object attributes for chunks. # CLI flag: -blocks-storage.bucket-store.chunks-cache.attributes-ttl [attributes_ttl: \u003cduration\u003e | default = 168h] # TTL for caching individual chunks subranges. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-ttl [subrange_ttl: \u003cduration\u003e | default = 24h] metadata_cache: # Backend for metadata cache, if not empty. Supported values: memcached, # redis, and '' (disable). # CLI flag: -blocks-storage.bucket-store.metadata-cache.backend [backend: \u003cstring\u003e | default = \"\"] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. If # set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should run. # If more keys are specified, internally keys are split into multiple # batches and fetched concurrently, honoring the max concurrency. If set # to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit breaker # becomes half-open. If set to 0, by default open duration is 60 # seconds. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. # If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching is # when data is stored in memory instead of fetching data each time. See # https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit breaker # becomes half-open. If set to 0, by default open duration is 60 # seconds. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # How long to cache list of tenants in the bucket. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl [tenants_list_ttl: \u003cduration\u003e | default = 15m] # How long to cache list of blocks for each tenant. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenant-blocks-list-ttl [tenant_blocks_list_ttl: \u003cduration\u003e | default = 5m] # How long to cache list of chunks for a block. # CLI flag: -blocks-storage.bucket-store.metadata-cache.chunks-list-ttl [chunks_list_ttl: \u003cduration\u003e | default = 24h] # How long to cache information that block metafile exists. Also used for # user deletion mark file. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-exists-ttl [metafile_exists_ttl: \u003cduration\u003e | default = 2h] # How long to cache information that block metafile doesn't exist. Also used # for user deletion mark file. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-doesnt-exist-ttl [metafile_doesnt_exist_ttl: \u003cduration\u003e | default = 5m] # How long to cache content of the metafile. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-content-ttl [metafile_content_ttl: \u003cduration\u003e | default = 24h] # Maximum size of metafile content to cache in bytes. Caching will be # skipped if the content exceeds this size. This is useful to avoid network # round trip for large content if the configured caching backend has an hard # limit on cached items size (in this case, you should set this limit to the # same limit in the caching backend). # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-max-size-bytes [metafile_max_size_bytes: \u003cint\u003e | default = 1048576] # How long to cache attributes of the block metafile. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-attributes-ttl [metafile_attributes_ttl: \u003cduration\u003e | default = 168h] # How long to cache attributes of the block index. # CLI flag: -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl [block_index_attributes_ttl: \u003cduration\u003e | default = 168h] # How long to cache content of the bucket index. # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl [bucket_index_content_ttl: \u003cduration\u003e | default = 5m] # Maximum size of bucket index content to cache in bytes. Caching will be # skipped if the content exceeds this size. This is useful to avoid network # round trip for large content if the configured caching backend has an hard # limit on cached items size (in this case, you should set this limit to the # same limit in the caching backend). # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-max-size-bytes [bucket_index_max_size_bytes: \u003cint\u003e | default = 1048576] # Duration after which the blocks marked for deletion will be filtered out # while fetching blocks. The idea of ignore-deletion-marks-delay is to ignore # blocks that are marked for deletion with some delay. This ensures store can # still serve blocks that are meant to be deleted but do not have a # replacement yet. Default is 6h, half of the default value for # -compactor.deletion-delay. # CLI flag: -blocks-storage.bucket-store.ignore-deletion-marks-delay [ignore_deletion_mark_delay: \u003cduration\u003e | default = 6h] # The blocks created since `now() - ignore_blocks_within` will not be synced. # This should be used together with `-querier.query-store-after` to filter out # the blocks that are too new to be queried. A reasonable value for this flag # would be `-querier.query-store-after - # blocks-storage.bucket-store.bucket-index.max-stale-period` to give some # buffer. 0 to disable. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: \u003cduration\u003e | default = 0s] bucket_index: # True to enable querier and store-gateway to discover blocks in the storage # via bucket index instead of bucket scanning. # CLI flag: -blocks-storage.bucket-store.bucket-index.enabled [enabled: \u003cboolean\u003e | default = false] # How frequently a bucket index, which previously failed to load, should be # tried to load again. This option is used only by querier. # CLI flag: -blocks-storage.bucket-store.bucket-index.update-on-error-interval [update_on_error_interval: \u003cduration\u003e | default = 1m] # How long a unused bucket index should be cached. Once this timeout # expires, the unused bucket index is removed from the in-memory cache. This # option is used only by querier. # CLI flag: -blocks-storage.bucket-store.bucket-index.idle-timeout [idle_timeout: \u003cduration\u003e | default = 1h] # The maximum allowed age of a bucket index (last updated) before queries # start failing because the bucket index is too old. The bucket index is # periodically updated by the compactor, while this check is enforced in the # querier (at query time). # CLI flag: -blocks-storage.bucket-store.bucket-index.max-stale-period [max_stale_period: \u003cduration\u003e | default = 1h] # One of concurrent, recursive, bucket_index. When set to concurrent, stores # will concurrently issue one call per directory to discover active blocks in # the bucket. The recursive strategy iterates through all objects in the # bucket, recursively traversing into each directory. This avoids N+1 calls at # the expense of having slower bucket iterations. bucket_index strategy can be # used in Compactor only and utilizes the existing bucket index to fetch block # IDs to sync. This avoids iterating the bucket but can be impacted by delays # of cleaner creating bucket index. # CLI flag: -blocks-storage.bucket-store.block-discovery-strategy [block_discovery_strategy: \u003cstring\u003e | default = \"concurrent\"] # Max size - in bytes - of a chunks pool, used to reduce memory allocations. # The pool is shared across all tenants. 0 to disable the limit. # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes [max_chunk_pool_bytes: \u003cint\u003e | default = 2147483648] # If enabled, store-gateway will lazily memory-map an index-header only once # required by a query. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-enabled [index_header_lazy_loading_enabled: \u003cboolean\u003e | default = false] # If index-header lazy loading is enabled and this setting is \u003e 0, the # store-gateway will release memory-mapped index-headers after 'idle timeout' # inactivity. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout [index_header_lazy_loading_idle_timeout: \u003cduration\u003e | default = 20m] # If true, Store Gateway will estimate postings size and try to lazily expand # postings if it downloads less data than expanding all postings. # CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled [lazy_expanded_postings_enabled: \u003cboolean\u003e | default = false] # Controls how many series to fetch per batch in Store Gateway. Default value # is 10000. # CLI flag: -blocks-storage.bucket-store.series-batch-size [series_batch_size: \u003cint\u003e | default = 10000] token_bucket_bytes_limiter: # Token bucket bytes limiter mode. Supported values are: disabled, dryrun, # enabled # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.mode [mode: \u003cstring\u003e | default = \"disabled\"] # Instance token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.instance-token-bucket-size [instance_token_bucket_size: \u003cint\u003e | default = 859832320] # User token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.user-token-bucket-size [user_token_bucket_size: \u003cint\u003e | default = 644874240] # Request token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.request-token-bucket-size [request_token_bucket_size: \u003cint\u003e | default = 4194304] tsdb: # Local directory to store TSDBs in the ingesters. # CLI flag: -blocks-storage.tsdb.dir [dir: \u003cstring\u003e | default = \"tsdb\"] # TSDB blocks range period. # CLI flag: -blocks-storage.tsdb.block-ranges-period [block_ranges_period: \u003clist of duration\u003e | default = 2h0m0s] # TSDB blocks retention in the ingester before a block is removed. This should # be larger than the block_ranges_period and large enough to give # store-gateways and queriers enough time to discover newly uploaded blocks. # CLI flag: -blocks-storage.tsdb.retention-period [retention_period: \u003cduration\u003e | default = 6h] # How frequently the TSDB blocks are scanned and new ones are shipped to the # storage. 0 means shipping is disabled. # CLI flag: -blocks-storage.tsdb.ship-interval [ship_interval: \u003cduration\u003e | default = 1m] # Maximum number of tenants concurrently shipping blocks to the storage. # CLI flag: -blocks-storage.tsdb.ship-concurrency [ship_concurrency: \u003cint\u003e | default = 10] # How frequently does Cortex try to compact TSDB head. Block is only created # if data covers smallest block range. Must be greater than 0 and max 30 # minutes. Note that up to 50% jitter is added to the value for the first # compaction to avoid ingesters compacting concurrently. # CLI flag: -blocks-storage.tsdb.head-compaction-interval [head_compaction_interval: \u003cduration\u003e | default = 1m] # Maximum number of tenants concurrently compacting TSDB head into a new block # CLI flag: -blocks-storage.tsdb.head-compaction-concurrency [head_compaction_concurrency: \u003cint\u003e | default = 5] # If TSDB head is idle for this duration, it is compacted. Note that up to 25% # jitter is added to the value to avoid ingesters compacting concurrently. 0 # means disabled. # CLI flag: -blocks-storage.tsdb.head-compaction-idle-timeout [head_compaction_idle_timeout: \u003cduration\u003e | default = 1h] # The write buffer size used by the head chunks mapper. Lower values reduce # memory utilisation on clusters with a large number of tenants at the cost of # increased disk I/O operations. # CLI flag: -blocks-storage.tsdb.head-chunks-write-buffer-size-bytes [head_chunks_write_buffer_size_bytes: \u003cint\u003e | default = 4194304] # The number of shards of series to use in TSDB (must be a power of 2). # Reducing this will decrease memory footprint, but can negatively impact # performance. # CLI flag: -blocks-storage.tsdb.stripe-size [stripe_size: \u003cint\u003e | default = 16384] # Deprecated (use blocks-storage.tsdb.wal-compression-type instead): True to # enable TSDB WAL compression. # CLI flag: -blocks-storage.tsdb.wal-compression-enabled [wal_compression_enabled: \u003cboolean\u003e | default = false] # TSDB WAL type. Supported values are: 'snappy', 'zstd' and '' (disable # compression) # CLI flag: -blocks-storage.tsdb.wal-compression-type [wal_compression_type: \u003cstring\u003e | default = \"\"] # TSDB WAL segments files max size (bytes). # CLI flag: -blocks-storage.tsdb.wal-segment-size-bytes [wal_segment_size_bytes: \u003cint\u003e | default = 134217728] # True to flush blocks to storage on shutdown. If false, incomplete blocks # will be reused after restart. # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown [flush_blocks_on_shutdown: \u003cboolean\u003e | default = false] # If TSDB has not received any data for this duration, and all blocks from # TSDB have been shipped, TSDB is closed and deleted from local disk. If set # to positive value, this value should be equal or higher than # -querier.query-ingesters-within flag to make sure that TSDB is not closed # prematurely, which could cause partial query results. 0 or negative value # disables closing of idle TSDB. # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout [close_idle_tsdb_timeout: \u003cduration\u003e | default = 0s] # The size of the in-memory queue used before flushing chunks to the disk. # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: \u003cint\u003e | default = 0] # limit the number of concurrently opening TSDB's on startup # CLI flag: -blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup [max_tsdb_opening_concurrency_on_startup: \u003cint\u003e | default = 10] # Deprecated, use maxExemplars in limits instead. If the MaxExemplars value in # limits is set to zero, cortex will fallback on this value. This setting # enables support for exemplars in TSDB and sets the maximum number that will # be stored. 0 or less means disabled. # CLI flag: -blocks-storage.tsdb.max-exemplars [max_exemplars: \u003cint\u003e | default = 0] # True to enable snapshotting of in-memory TSDB data on disk when shutting # down. # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown [memory_snapshot_on_shutdown: \u003cboolean\u003e | default = false] # [EXPERIMENTAL] Configures the maximum number of samples per chunk that can # be out-of-order. # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max [out_of_order_cap_max: \u003cint\u003e | default = 32] # [EXPERIMENTAL] True to enable native histogram. # CLI flag: -blocks-storage.tsdb.enable-native-histograms [enable_native_histograms: \u003cboolean\u003e | default = false] # [EXPERIMENTAL] If enabled, ingesters will cache expanded postings when # querying blocks. Caching can be configured separately for the head and # compacted blocks. expanded_postings_cache: # If enabled, ingesters will cache expanded postings for the head block. # Only queries with with an equal matcher for metric __name__ are cached. head: # Whether the postings cache is enabled or not # CLI flag: -blocks-storage.expanded_postings_cache.head.enabled [enabled: \u003cboolean\u003e | default = false] # Max bytes for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.head.max-bytes [max_bytes: \u003cint\u003e | default = 10485760] # TTL for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.head.ttl [ttl: \u003cduration\u003e | default = 10m] # If enabled, ingesters will cache expanded postings for the compacted # blocks. The cache is shared between all blocks. blocks: # Whether the postings cache is enabled or not # CLI flag: -blocks-storage.expanded_postings_cache.block.enabled [enabled: \u003cboolean\u003e | default = false] # Max bytes for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.block.max-bytes [max_bytes: \u003cint\u003e | default = 10485760] # TTL for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.block.ttl [ttl: \u003cduration\u003e | default = 10m] compactor_config The compactor_config configures the compactor for the blocks storage.\n# List of compaction time ranges. # CLI flag: -compactor.block-ranges [block_ranges: \u003clist of duration\u003e | default = 2h0m0s,12h0m0s,24h0m0s] # Number of Go routines to use when syncing block index and chunks files from # the long term storage. # CLI flag: -compactor.block-sync-concurrency [block_sync_concurrency: \u003cint\u003e | default = 20] # Number of Go routines to use when syncing block meta files from the long term # storage. # CLI flag: -compactor.meta-sync-concurrency [meta_sync_concurrency: \u003cint\u003e | default = 20] # Minimum age of fresh (non-compacted) blocks before they are being processed. # Malformed blocks older than the maximum of consistency-delay and 48h0m0s will # be removed. # CLI flag: -compactor.consistency-delay [consistency_delay: \u003cduration\u003e | default = 0s] # Data directory in which to cache blocks and process compactions # CLI flag: -compactor.data-dir [data_dir: \u003cstring\u003e | default = \"./data\"] # The frequency at which the compaction runs # CLI flag: -compactor.compaction-interval [compaction_interval: \u003cduration\u003e | default = 1h] # How many times to retry a failed compaction within a single compaction run. # CLI flag: -compactor.compaction-retries [compaction_retries: \u003cint\u003e | default = 3] # Max number of concurrent compactions running. # CLI flag: -compactor.compaction-concurrency [compaction_concurrency: \u003cint\u003e | default = 1] # How frequently compactor should run blocks cleanup and maintenance, as well as # update the bucket index. # CLI flag: -compactor.cleanup-interval [cleanup_interval: \u003cduration\u003e | default = 15m] # Max number of tenants for which blocks cleanup and maintenance should run # concurrently. # CLI flag: -compactor.cleanup-concurrency [cleanup_concurrency: \u003cint\u003e | default = 20] # Time before a block marked for deletion is deleted from bucket. If not 0, # blocks will be marked for deletion and compactor component will permanently # delete blocks marked for deletion from the bucket. If 0, blocks will be # deleted straight away. Note that deleting blocks immediately can cause query # failures. # CLI flag: -compactor.deletion-delay [deletion_delay: \u003cduration\u003e | default = 12h] # For tenants marked for deletion, this is time between deleting of last block, # and doing final cleanup (marker files, debug files) of the tenant. # CLI flag: -compactor.tenant-cleanup-delay [tenant_cleanup_delay: \u003cduration\u003e | default = 6h] # When enabled, mark blocks containing index with out-of-order chunks for no # compact instead of halting the compaction. # CLI flag: -compactor.skip-blocks-with-out-of-order-chunks-enabled [skip_blocks_with_out_of_order_chunks_enabled: \u003cboolean\u003e | default = false] # Number of goroutines to use when fetching/uploading block files from object # storage. # CLI flag: -compactor.block-files-concurrency [block_files_concurrency: \u003cint\u003e | default = 10] # Number of goroutines to use when fetching blocks from object storage when # compacting. # CLI flag: -compactor.blocks-fetch-concurrency [blocks_fetch_concurrency: \u003cint\u003e | default = 3] # When enabled, at compactor startup the bucket will be scanned and all found # deletion marks inside the block location will be copied to the markers global # location too. This option can (and should) be safely disabled as soon as the # compactor has successfully run at least once. # CLI flag: -compactor.block-deletion-marks-migration-enabled [block_deletion_marks_migration_enabled: \u003cboolean\u003e | default = false] # Comma separated list of tenants that can be compacted. If specified, only # these tenants will be compacted by compactor, otherwise all tenants can be # compacted. Subject to sharding. # CLI flag: -compactor.enabled-tenants [enabled_tenants: \u003cstring\u003e | default = \"\"] # Comma separated list of tenants that cannot be compacted by this compactor. If # specified, and compactor would normally pick given tenant for compaction (via # -compactor.enabled-tenants or sharding), it will be ignored instead. # CLI flag: -compactor.disabled-tenants [disabled_tenants: \u003cstring\u003e | default = \"\"] # Shard tenants across multiple compactor instances. Sharding is required if you # run multiple compactor instances, in order to coordinate compactions and avoid # race conditions leading to the same tenant blocks simultaneously compacted by # different instances. # CLI flag: -compactor.sharding-enabled [sharding_enabled: \u003cboolean\u003e | default = false] # The sharding strategy to use. Supported values are: default, shuffle-sharding. # CLI flag: -compactor.sharding-strategy [sharding_strategy: \u003cstring\u003e | default = \"default\"] sharding_ring: kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -compactor.ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -compactor.ring.prefix [prefix: \u003cstring\u003e | default = \"collectors/\"] dynamodb: # Region to access dynamodb. # CLI flag: -compactor.ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -compactor.ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -compactor.ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -compactor.ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -compactor.ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: compactor.ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: compactor.ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -compactor.ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -compactor.ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -compactor.ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -compactor.ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -compactor.ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 5s] # The heartbeat timeout after which compactors are considered unhealthy within # the ring. 0 = never (timeout disabled). # CLI flag: -compactor.ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # Minimum time to wait for ring stability at startup. 0 to disable. # CLI flag: -compactor.ring.wait-stability-min-duration [wait_stability_min_duration: \u003cduration\u003e | default = 1m] # Maximum time to wait for ring stability at startup. If the compactor ring # keeps changing after this period of time, the compactor will start anyway. # CLI flag: -compactor.ring.wait-stability-max-duration [wait_stability_max_duration: \u003cduration\u003e | default = 5m] # Name of network interface to read address from. # CLI flag: -compactor.ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # File path where tokens are stored. If empty, tokens are not stored at # shutdown and restored at startup. # CLI flag: -compactor.ring.tokens-file-path [tokens_file_path: \u003cstring\u003e | default = \"\"] # Unregister the compactor during shutdown if true. # CLI flag: -compactor.ring.unregister-on-shutdown [unregister_on_shutdown: \u003cboolean\u003e | default = true] # Timeout for waiting on compactor to become ACTIVE in the ring. # CLI flag: -compactor.ring.wait-active-instance-timeout [wait_active_instance_timeout: \u003cduration\u003e | default = 10m] # The compaction strategy to use. Supported values are: default, partitioning. # CLI flag: -compactor.compaction-mode [compaction_mode: \u003cstring\u003e | default = \"default\"] # How long block visit marker file should be considered as expired and able to # be picked up by compactor again. # CLI flag: -compactor.block-visit-marker-timeout [block_visit_marker_timeout: \u003cduration\u003e | default = 5m] # How frequently block visit marker file should be updated duration compaction. # CLI flag: -compactor.block-visit-marker-file-update-interval [block_visit_marker_file_update_interval: \u003cduration\u003e | default = 1m] # How long cleaner visit marker file should be considered as expired and able to # be picked up by cleaner again. The value should be smaller than # -compactor.cleanup-interval # CLI flag: -compactor.cleaner-visit-marker-timeout [cleaner_visit_marker_timeout: \u003cduration\u003e | default = 10m] # How frequently cleaner visit marker file should be updated when cleaning user. # CLI flag: -compactor.cleaner-visit-marker-file-update-interval [cleaner_visit_marker_file_update_interval: \u003cduration\u003e | default = 5m] # When enabled, index verification will ignore out of order label names. # CLI flag: -compactor.accept-malformed-index [accept_malformed_index: \u003cboolean\u003e | default = false] # When enabled, caching bucket will be used for compactor, except cleaner # service, which serves as the source of truth for block status # CLI flag: -compactor.caching-bucket-enabled [caching_bucket_enabled: \u003cboolean\u003e | default = false] configs_config The configs_config configures the Cortex Configs DB and API.\ndatabase: # URI where the database can be found (for dev you can use memory://) # CLI flag: -configs.database.uri [uri: \u003cstring\u003e | default = \"postgres://postgres@configs-db.weave.local/configs?sslmode=disable\"] # Path where the database migration files can be found # CLI flag: -configs.database.migrations-dir [migrations_dir: \u003cstring\u003e | default = \"\"] # File containing password (username goes in URI) # CLI flag: -configs.database.password-file [password_file: \u003cstring\u003e | default = \"\"] api: notifications: # Disable Email notifications for Alertmanager. # CLI flag: -configs.notifications.disable-email [disable_email: \u003cboolean\u003e | default = false] # Disable WebHook notifications for Alertmanager. # CLI flag: -configs.notifications.disable-webhook [disable_webhook: \u003cboolean\u003e | default = false] configstore_config The configstore_config configures the config database storing rules and alerts, and is used by the Cortex alertmanager. The supported CLI flags \u003cprefix\u003e used to reference this config block are:\nalertmanager-storage ruler-storage # URL of configs API server. # CLI flag: -\u003cprefix\u003e.configs.url [configs_api_url: \u003curl\u003e | default = ] # Timeout for requests to Weave Cloud configs service. # CLI flag: -\u003cprefix\u003e.configs.client-timeout [client_timeout: \u003cduration\u003e | default = 5s] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -\u003cprefix\u003e.configs.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -\u003cprefix\u003e.configs.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -\u003cprefix\u003e.configs.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -\u003cprefix\u003e.configs.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -\u003cprefix\u003e.configs.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] consul_config The consul_config configures the consul client. The supported CLI flags \u003cprefix\u003e used to reference this config block are:\nno prefix alertmanager.sharding-ring compactor.ring distributor.ha-tracker distributor.ring ruler.ring store-gateway.sharding-ring # Hostname and port of Consul. # CLI flag: -\u003cprefix\u003e.consul.hostname [host: \u003cstring\u003e | default = \"localhost:8500\"] # ACL Token used to interact with Consul. # CLI flag: -\u003cprefix\u003e.consul.acl-token [acl_token: \u003cstring\u003e | default = \"\"] # HTTP timeout when talking to Consul. # CLI flag: -\u003cprefix\u003e.consul.client-timeout [http_client_timeout: \u003cduration\u003e | default = 20s] # Enable consistent reads to Consul. # CLI flag: -\u003cprefix\u003e.consul.consistent-reads [consistent_reads: \u003cboolean\u003e | default = false] # Rate limit when watching key or prefix in Consul, in requests per second. 0 # disables the rate limit. # CLI flag: -\u003cprefix\u003e.consul.watch-rate-limit [watch_rate_limit: \u003cfloat\u003e | default = 1] # Burst size used in rate limit. Values less than 1 are treated as 1. # CLI flag: -\u003cprefix\u003e.consul.watch-burst-size [watch_burst_size: \u003cint\u003e | default = 1] # Enable TLS. # CLI flag: -\u003cprefix\u003e.consul.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -\u003cprefix\u003e.consul.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -\u003cprefix\u003e.consul.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -\u003cprefix\u003e.consul.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -\u003cprefix\u003e.consul.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -\u003cprefix\u003e.consul.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] distributor_config The distributor_config configures the Cortex distributor.\npool: # How frequently to clean up clients for ingesters that have gone away. # CLI flag: -distributor.client-cleanup-period [client_cleanup_period: \u003cduration\u003e | default = 15s] # Run a health check on each ingester client during periodic cleanup. # CLI flag: -distributor.health-check-ingesters [health_check_ingesters: \u003cboolean\u003e | default = true] ha_tracker: # Enable the distributors HA tracker so that it can accept samples from # Prometheus HA replicas gracefully (requires labels). # CLI flag: -distributor.ha-tracker.enable [enable_ha_tracker: \u003cboolean\u003e | default = false] # Update the timestamp in the KV store for a given cluster/replica only after # this amount of time has passed since the current stored timestamp. # CLI flag: -distributor.ha-tracker.update-timeout [ha_tracker_update_timeout: \u003cduration\u003e | default = 15s] # Maximum jitter applied to the update timeout, in order to spread the HA # heartbeats over time. # CLI flag: -distributor.ha-tracker.update-timeout-jitter-max [ha_tracker_update_timeout_jitter_max: \u003cduration\u003e | default = 5s] # If we don't receive any samples from the accepted replica for a cluster in # this amount of time we will failover to the next replica we receive a sample # from. This value must be greater than the update timeout # CLI flag: -distributor.ha-tracker.failover-timeout [ha_tracker_failover_timeout: \u003cduration\u003e | default = 30s] # Backend storage to use for the ring. Please be aware that memberlist is not # supported by the HA tracker since gossip propagation is too slow for HA # purposes. kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -distributor.ha-tracker.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -distributor.ha-tracker.prefix [prefix: \u003cstring\u003e | default = \"ha-tracker/\"] dynamodb: # Region to access dynamodb. # CLI flag: -distributor.ha-tracker.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -distributor.ha-tracker.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -distributor.ha-tracker.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -distributor.ha-tracker.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -distributor.ha-tracker.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: distributor.ha-tracker [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: distributor.ha-tracker [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -distributor.ha-tracker.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -distributor.ha-tracker.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -distributor.ha-tracker.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -distributor.ha-tracker.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # remote_write API max receive message size (bytes). # CLI flag: -distributor.max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # Maximum OTLP request size in bytes that the Distributor can accept. # CLI flag: -distributor.otlp-max-recv-msg-size [otlp_max_recv_msg_size: \u003cint\u003e | default = 104857600] # Timeout for downstream ingesters. # CLI flag: -distributor.remote-timeout [remote_timeout: \u003cduration\u003e | default = 2s] # Time to wait before sending more than the minimum successful query requests. # CLI flag: -distributor.extra-query-delay [extra_queue_delay: \u003cduration\u003e | default = 0s] # The sharding strategy to use. Supported values are: default, shuffle-sharding. # CLI flag: -distributor.sharding-strategy [sharding_strategy: \u003cstring\u003e | default = \"default\"] # Distribute samples based on all labels, as opposed to solely by user and # metric name. # CLI flag: -distributor.shard-by-all-labels [shard_by_all_labels: \u003cboolean\u003e | default = false] # Try writing to an additional ingester in the presence of an ingester not in # the ACTIVE state. It is useful to disable this along with # -ingester.unregister-on-shutdown=false in order to not spread samples to extra # ingesters during rolling restarts with consistent naming. # CLI flag: -distributor.extend-writes [extend_writes: \u003cboolean\u003e | default = true] # EXPERIMENTAL: If enabled, sign the write request between distributors and # ingesters. # CLI flag: -distributor.sign-write-requests [sign_write_requests: \u003cboolean\u003e | default = false] ring: kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -distributor.ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -distributor.ring.prefix [prefix: \u003cstring\u003e | default = \"collectors/\"] dynamodb: # Region to access dynamodb. # CLI flag: -distributor.ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -distributor.ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -distributor.ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -distributor.ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -distributor.ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: distributor.ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: distributor.ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -distributor.ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -distributor.ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -distributor.ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -distributor.ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -distributor.ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 5s] # The heartbeat timeout after which distributors are considered unhealthy # within the ring. 0 = never (timeout disabled). # CLI flag: -distributor.ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # Name of network interface to read address from. # CLI flag: -distributor.ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] instance_limits: # Max ingestion rate (samples/sec) that this distributor will accept. This # limit is per-distributor, not per-tenant. Additional push requests will be # rejected. Current ingestion rate is computed as exponentially weighted # moving average, updated every second. 0 = unlimited. # CLI flag: -distributor.instance-limits.max-ingestion-rate [max_ingestion_rate: \u003cfloat\u003e | default = 0] # Max inflight push requests that this distributor can handle. This limit is # per-distributor, not per-tenant. Additional requests will be rejected. 0 = # unlimited. # CLI flag: -distributor.instance-limits.max-inflight-push-requests [max_inflight_push_requests: \u003cint\u003e | default = 0] # Max inflight ingester client requests that this distributor can handle. This # limit is per-distributor, not per-tenant. Additional requests will be # rejected. 0 = unlimited. # CLI flag: -distributor.instance-limits.max-inflight-client-requests [max_inflight_client_requests: \u003cint\u003e | default = 0] otlp: # If true, all resource attributes are converted to labels. # CLI flag: -distributor.otlp.convert-all-attributes [convert_all_attributes: \u003cboolean\u003e | default = false] # If true, a target_info metric is not ingested. (refer to: # https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#supporting-target-metadata-in-both-push-based-and-pull-based-systems) # CLI flag: -distributor.otlp.disable-target-info [disable_target_info: \u003cboolean\u003e | default = false] etcd_config The etcd_config configures the etcd client. The supported CLI flags \u003cprefix\u003e used to reference this config block are:\nno prefix alertmanager.sharding-ring compactor.ring distributor.ha-tracker distributor.ring ruler.ring store-gateway.sharding-ring # The etcd endpoints to connect to. # CLI flag: -\u003cprefix\u003e.etcd.endpoints [endpoints: \u003clist of string\u003e | default = []] # The dial timeout for the etcd connection. # CLI flag: -\u003cprefix\u003e.etcd.dial-timeout [dial_timeout: \u003cduration\u003e | default = 10s] # The maximum number of retries to do for failed ops. # CLI flag: -\u003cprefix\u003e.etcd.max-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS. # CLI flag: -\u003cprefix\u003e.etcd.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -\u003cprefix\u003e.etcd.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -\u003cprefix\u003e.etcd.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -\u003cprefix\u003e.etcd.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -\u003cprefix\u003e.etcd.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -\u003cprefix\u003e.etcd.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # Etcd username. # CLI flag: -\u003cprefix\u003e.etcd.username [username: \u003cstring\u003e | default = \"\"] # Etcd password. # CLI flag: -\u003cprefix\u003e.etcd.password [password: \u003cstring\u003e | default = \"\"] # Send Keepalive pings with no streams. # CLI flag: -\u003cprefix\u003e.etcd.ping-without-stream-allowed [ping-without-stream-allowed: \u003cboolean\u003e | default = true] fifo_cache_config The fifo_cache_config configures the local in-memory cache.\n# Maximum memory size of the cache in bytes. A unit suffix (KB, MB, GB) may be # applied. # CLI flag: -frontend.fifocache.max-size-bytes [max_size_bytes: \u003cstring\u003e | default = \"\"] # Maximum number of entries in the cache. # CLI flag: -frontend.fifocache.max-size-items [max_size_items: \u003cint\u003e | default = 0] # The expiry duration for the cache. # CLI flag: -frontend.fifocache.duration [validity: \u003cduration\u003e | default = 0s] # Deprecated (use max-size-items or max-size-bytes instead): The number of # entries to cache. # CLI flag: -frontend.fifocache.size [size: \u003cint\u003e | default = 0] flusher_config The flusher_config configures the WAL flusher target, used to manually run one-time flushes when scaling down ingesters.\n# Stop Cortex after flush has finished. If false, Cortex process will keep # running, doing nothing. # CLI flag: -flusher.exit-after-flush [exit_after_flush: \u003cboolean\u003e | default = true] frontend_worker_config The frontend_worker_config configures the worker - running within the Cortex querier - picking up and executing queries enqueued by the query-frontend or query-scheduler.\n# Address of query frontend service, in host:port format. If # -querier.scheduler-address is set as well, querier will use scheduler instead. # Only one of -querier.frontend-address or -querier.scheduler-address can be # set. If neither is set, queries are only received via HTTP endpoint. # CLI flag: -querier.frontend-address [frontend_address: \u003cstring\u003e | default = \"\"] # Hostname (and port) of scheduler that querier will periodically resolve, # connect to and receive queries from. Only one of -querier.frontend-address or # -querier.scheduler-address can be set. If neither is set, queries are only # received via HTTP endpoint. # CLI flag: -querier.scheduler-address [scheduler_address: \u003cstring\u003e | default = \"\"] # How often to query DNS for query-frontend or query-scheduler address. # CLI flag: -querier.dns-lookup-period [dns_lookup_duration: \u003cduration\u003e | default = 10s] # Number of simultaneous queries to process per query-frontend or # query-scheduler. # CLI flag: -querier.worker-parallelism [parallelism: \u003cint\u003e | default = 10] # Force worker concurrency to match the -querier.max-concurrent option. # Overrides querier.worker-parallelism. # CLI flag: -querier.worker-match-max-concurrent [match_max_concurrent: \u003cboolean\u003e | default = false] # Querier ID, sent to frontend service to identify requests from the same # querier. Defaults to hostname. # CLI flag: -querier.id [id: \u003cstring\u003e | default = \"\"] grpc_client_config: # gRPC client max receive message size (bytes). # CLI flag: -querier.frontend-client.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # gRPC client max send message size (bytes). # CLI flag: -querier.frontend-client.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 16777216] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy', 'snappy-block' ,'zstd' and '' (disable compression) # CLI flag: -querier.frontend-client.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # Rate limit for gRPC client; 0 means disabled. # CLI flag: -querier.frontend-client.grpc-client-rate-limit [rate_limit: \u003cfloat\u003e | default = 0] # Rate limit burst for gRPC client. # CLI flag: -querier.frontend-client.grpc-client-rate-limit-burst [rate_limit_burst: \u003cint\u003e | default = 0] # Enable backoff and retry when we hit ratelimits. # CLI flag: -querier.frontend-client.backoff-on-ratelimits [backoff_on_ratelimits: \u003cboolean\u003e | default = false] backoff_config: # Minimum delay when backing off. # CLI flag: -querier.frontend-client.backoff-min-period [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -querier.frontend-client.backoff-max-period [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -querier.frontend-client.backoff-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -querier.frontend-client.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -querier.frontend-client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -querier.frontend-client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -querier.frontend-client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -querier.frontend-client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -querier.frontend-client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] ingester_config The ingester_config configures the Cortex ingester.\nlifecycler: ring: kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -ring.prefix [prefix: \u003cstring\u003e | default = \"collectors/\"] dynamodb: # Region to access dynamodb. # CLI flag: -dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # The heartbeat timeout after which ingesters are skipped for reads/writes. # 0 = never (timeout disabled). # CLI flag: -ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # The number of ingesters to write to and read from. # CLI flag: -distributor.replication-factor [replication_factor: \u003cint\u003e | default = 3] # True to enable the zone-awareness and replicate ingested samples across # different availability zones. # CLI flag: -distributor.zone-awareness-enabled [zone_awareness_enabled: \u003cboolean\u003e | default = false] # Comma-separated list of zones to exclude from the ring. Instances in # excluded zones will be filtered out from the ring. # CLI flag: -distributor.excluded-zones [excluded_zones: \u003cstring\u003e | default = \"\"] # Set to true to enable ring detailed metrics. These metrics provide # detailed information, such as token count and ownership per tenant. # Disabling them can significantly decrease the number of metrics emitted by # the distributors. # CLI flag: -ring.detailed-metrics-enabled [detailed_metrics_enabled: \u003cboolean\u003e | default = true] # Number of tokens for each ingester. # CLI flag: -ingester.num-tokens [num_tokens: \u003cint\u003e | default = 128] # EXPERIMENTAL: Algorithm used to generate new ring tokens. Supported Values: # random,minimize-spread # CLI flag: -ingester.tokens-generator-strategy [tokens_generator_strategy: \u003cstring\u003e | default = \"random\"] # Period at which to heartbeat to consul. 0 = disabled. # CLI flag: -ingester.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 5s] # Observe tokens after generating to resolve collisions. Useful when using # gossiping ring. # CLI flag: -ingester.observe-period [observe_period: \u003cduration\u003e | default = 0s] # Period to wait for a claim from another member; will join automatically # after this. # CLI flag: -ingester.join-after [join_after: \u003cduration\u003e | default = 0s] # Minimum duration to wait after the internal readiness checks have passed but # before succeeding the readiness endpoint. This is used to slowdown # deployment controllers (eg. Kubernetes) after an instance is ready and # before they proceed with a rolling update, to give the rest of the cluster # instances enough time to receive ring updates. # CLI flag: -ingester.min-ready-duration [min_ready_duration: \u003cduration\u003e | default = 15s] # Name of network interface to read address from. # CLI flag: -ingester.lifecycler.interface [interface_names: \u003clist of string\u003e | default = [eth0 en0]] # Duration to sleep for before exiting, to ensure metrics are scraped. # CLI flag: -ingester.final-sleep [final_sleep: \u003cduration\u003e | default = 30s] # File path where tokens are stored. If empty, tokens are not stored at # shutdown and restored at startup. # CLI flag: -ingester.tokens-file-path [tokens_file_path: \u003cstring\u003e | default = \"\"] # The availability zone where this instance is running. # CLI flag: -ingester.availability-zone [availability_zone: \u003cstring\u003e | default = \"\"] # Unregister from the ring upon clean shutdown. It can be useful to disable # for rolling restarts with consistent naming in conjunction with # -distributor.extend-writes=false. # CLI flag: -ingester.unregister-on-shutdown [unregister_on_shutdown: \u003cboolean\u003e | default = true] # When enabled the readiness probe succeeds only after all instances are # ACTIVE and healthy in the ring, otherwise only the instance itself is # checked. This option should be disabled if in your cluster multiple # instances can be rolled out simultaneously, otherwise rolling updates may be # slowed down. # CLI flag: -ingester.readiness-check-ring-health [readiness_check_ring_health: \u003cboolean\u003e | default = true] # Period at which metadata we have not seen will remain in memory before being # deleted. # CLI flag: -ingester.metadata-retain-period [metadata_retain_period: \u003cduration\u003e | default = 10m] # Period with which to update the per-user ingestion rates. # CLI flag: -ingester.rate-update-period [rate_update_period: \u003cduration\u003e | default = 15s] # Period with which to update the per-user tsdb config. # CLI flag: -ingester.user-tsdb-configs-update-period [user_tsdb_configs_update_period: \u003cduration\u003e | default = 15s] # Enable tracking of active series and export them as metrics. # CLI flag: -ingester.active-series-metrics-enabled [active_series_metrics_enabled: \u003cboolean\u003e | default = true] # How often to update active series metrics. # CLI flag: -ingester.active-series-metrics-update-period [active_series_metrics_update_period: \u003cduration\u003e | default = 1m] # After what time a series is considered to be inactive. # CLI flag: -ingester.active-series-metrics-idle-timeout [active_series_metrics_idle_timeout: \u003cduration\u003e | default = 10m] # Enable uploading compacted blocks. # CLI flag: -ingester.upload-compacted-blocks-enabled [upload_compacted_blocks_enabled: \u003cboolean\u003e | default = true] instance_limits: # Max ingestion rate (samples/sec) that ingester will accept. This limit is # per-ingester, not per-tenant. Additional push requests will be rejected. # Current ingestion rate is computed as exponentially weighted moving average, # updated every second. This limit only works when using blocks engine. 0 = # unlimited. # CLI flag: -ingester.instance-limits.max-ingestion-rate [max_ingestion_rate: \u003cfloat\u003e | default = 0] # Max users that this ingester can hold. Requests from additional users will # be rejected. This limit only works when using blocks engine. 0 = unlimited. # CLI flag: -ingester.instance-limits.max-tenants [max_tenants: \u003cint\u003e | default = 0] # Max series that this ingester can hold (across all tenants). Requests to # create additional series will be rejected. This limit only works when using # blocks engine. 0 = unlimited. # CLI flag: -ingester.instance-limits.max-series [max_series: \u003cint\u003e | default = 0] # Max inflight push requests that this ingester can handle (across all # tenants). Additional requests will be rejected. 0 = unlimited. # CLI flag: -ingester.instance-limits.max-inflight-push-requests [max_inflight_push_requests: \u003cint\u003e | default = 0] # Max inflight query requests that this ingester can handle (across all # tenants). Additional requests will be rejected. 0 = unlimited. # CLI flag: -ingester.instance-limits.max-inflight-query-requests [max_inflight_query_requests: \u003cint\u003e | default = 0] # Comma-separated list of metric names, for which # -ingester.max-series-per-metric and -ingester.max-global-series-per-metric # limits will be ignored. Does not affect max-series-per-user or # max-global-series-per-metric limits. # CLI flag: -ingester.ignore-series-limit-for-metric-names [ignore_series_limit_for_metric_names: \u003cstring\u003e | default = \"\"] # Customize the message contained in limit errors # CLI flag: -ingester.admin-limit-message [admin_limit_message: \u003cstring\u003e | default = \"please contact administrator to raise it\"] # Experimental: Enable string interning for metrics labels. # CLI flag: -ingester.labels-string-interning-enabled [labels_string_interning_enabled: \u003cboolean\u003e | default = false] # Disable trimming of matching series chunks based on query Start and End time. # When disabled, the result may contain samples outside the queried time range # but select performances may be improved. Note that certain query results might # change by changing this option. # CLI flag: -ingester.disable-chunk-trimming [disable_chunk_trimming: \u003cboolean\u003e | default = false] ingester_client_config The ingester_client_config configures how the Cortex distributors connect to the ingesters.\ngrpc_client_config: # gRPC client max receive message size (bytes). # CLI flag: -ingester.client.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # gRPC client max send message size (bytes). # CLI flag: -ingester.client.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 16777216] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy', 'snappy-block' ,'zstd' and '' (disable compression) # CLI flag: -ingester.client.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"snappy-block\"] # Rate limit for gRPC client; 0 means disabled. # CLI flag: -ingester.client.grpc-client-rate-limit [rate_limit: \u003cfloat\u003e | default = 0] # Rate limit burst for gRPC client. # CLI flag: -ingester.client.grpc-client-rate-limit-burst [rate_limit_burst: \u003cint\u003e | default = 0] # Enable backoff and retry when we hit ratelimits. # CLI flag: -ingester.client.backoff-on-ratelimits [backoff_on_ratelimits: \u003cboolean\u003e | default = false] backoff_config: # Minimum delay when backing off. # CLI flag: -ingester.client.backoff-min-period [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -ingester.client.backoff-max-period [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -ingester.client.backoff-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -ingester.client.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -ingester.client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -ingester.client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -ingester.client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -ingester.client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -ingester.client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # EXPERIMENTAL: If enabled, gRPC clients perform health checks for each target # and fail the request if the target is marked as unhealthy. healthcheck_config: # The number of consecutive failed health checks required before considering # a target unhealthy. 0 means disabled. # CLI flag: -ingester.client.unhealthy-threshold [unhealthy_threshold: \u003cint\u003e | default = 0] # The approximate amount of time between health checks of an individual # target. # CLI flag: -ingester.client.interval [interval: \u003cduration\u003e | default = 5s] # The amount of time during which no response from a target means a failed # health check. # CLI flag: -ingester.client.timeout [timeout: \u003cduration\u003e | default = 1s] # Max inflight push requests that this ingester client can handle. This limit is # per-ingester-client. Additional requests will be rejected. 0 = unlimited. # CLI flag: -ingester.client.max-inflight-push-requests [max_inflight_push_requests: \u003cint\u003e | default = 0] limits_config The limits_config configures default and per-tenant limits imposed by Cortex services (ie. distributor, ingester, …).\n# Per-user ingestion rate limit in samples per second. # CLI flag: -distributor.ingestion-rate-limit [ingestion_rate: \u003cfloat\u003e | default = 25000] # Whether the ingestion rate limit should be applied individually to each # distributor instance (local), or evenly shared across the cluster (global). # CLI flag: -distributor.ingestion-rate-limit-strategy [ingestion_rate_strategy: \u003cstring\u003e | default = \"local\"] # Per-user allowed ingestion burst size (in number of samples). # CLI flag: -distributor.ingestion-burst-size [ingestion_burst_size: \u003cint\u003e | default = 50000] # Flag to enable, for all users, handling of samples with external labels # identifying replicas in an HA Prometheus setup. # CLI flag: -distributor.ha-tracker.enable-for-all-users [accept_ha_samples: \u003cboolean\u003e | default = false] # [Experimental] Flag to enable handling of samples with mixed external labels # identifying replicas in an HA Prometheus setup. Supported only if # -distributor.ha-tracker.enable-for-all-users is true. # CLI flag: -experimental.distributor.ha-tracker.mixed-ha-samples [accept_mixed_ha_samples: \u003cboolean\u003e | default = false] # Prometheus label to look for in samples to identify a Prometheus HA cluster. # CLI flag: -distributor.ha-tracker.cluster [ha_cluster_label: \u003cstring\u003e | default = \"cluster\"] # Prometheus label to look for in samples to identify a Prometheus HA replica. # CLI flag: -distributor.ha-tracker.replica [ha_replica_label: \u003cstring\u003e | default = \"__replica__\"] # Maximum number of clusters that HA tracker will keep track of for single user. # 0 to disable the limit. # CLI flag: -distributor.ha-tracker.max-clusters [ha_max_clusters: \u003cint\u003e | default = 0] # This flag can be used to specify label names that to drop during sample # ingestion within the distributor and can be repeated in order to drop multiple # labels. # CLI flag: -distributor.drop-label [drop_labels: \u003clist of string\u003e | default = []] # Maximum length accepted for label names # CLI flag: -validation.max-length-label-name [max_label_name_length: \u003cint\u003e | default = 1024] # Maximum length accepted for label value. This setting also applies to the # metric name # CLI flag: -validation.max-length-label-value [max_label_value_length: \u003cint\u003e | default = 2048] # Maximum number of label names per series. # CLI flag: -validation.max-label-names-per-series [max_label_names_per_series: \u003cint\u003e | default = 30] # Maximum combined size in bytes of all labels and label values accepted for a # series. 0 to disable the limit. # CLI flag: -validation.max-labels-size-bytes [max_labels_size_bytes: \u003cint\u003e | default = 0] # Maximum length accepted for metric metadata. Metadata refers to Metric Name, # HELP and UNIT. # CLI flag: -validation.max-metadata-length [max_metadata_length: \u003cint\u003e | default = 1024] # Reject old samples. # CLI flag: -validation.reject-old-samples [reject_old_samples: \u003cboolean\u003e | default = false] # Maximum accepted sample age before rejecting. # CLI flag: -validation.reject-old-samples.max-age [reject_old_samples_max_age: \u003cduration\u003e | default = 2w] # Duration which table will be created/deleted before/after it's needed; we # won't accept sample from before this time. # CLI flag: -validation.create-grace-period [creation_grace_period: \u003cduration\u003e | default = 10m] # Enforce every metadata has a metric name. # CLI flag: -validation.enforce-metadata-metric-name [enforce_metadata_metric_name: \u003cboolean\u003e | default = true] # Enforce every sample has a metric name. # CLI flag: -validation.enforce-metric-name [enforce_metric_name: \u003cboolean\u003e | default = true] # The default tenant's shard size when the shuffle-sharding strategy is used. # Must be set both on ingesters and distributors. When this setting is specified # in the per-tenant overrides, a value of 0 disables shuffle sharding for the # tenant. # CLI flag: -distributor.ingestion-tenant-shard-size [ingestion_tenant_shard_size: \u003cint\u003e | default = 0] # List of metric relabel configurations. Note that in most situations, it is # more effective to use metrics relabeling directly in the Prometheus server, # e.g. remote_write.write_relabel_configs. [metric_relabel_configs: \u003crelabel_config...\u003e | default = []] # Limit on total number of positive and negative buckets allowed in a single # native histogram. The resolution of a histogram with more buckets will be # reduced until the number of buckets is within the limit. If the limit cannot # be reached, the sample will be discarded. 0 means no limit. Enforced at # Distributor. # CLI flag: -validation.max-native-histogram-buckets [max_native_histogram_buckets: \u003cint\u003e | default = 0] # Comma separated list of resource attributes that should be converted to # labels. # CLI flag: -distributor.promote-resource-attributes [promote_resource_attributes: \u003clist of string\u003e | default = ] # The maximum number of active series per user, per ingester. 0 to disable. # CLI flag: -ingester.max-series-per-user [max_series_per_user: \u003cint\u003e | default = 5000000] # The maximum number of active series per metric name, per ingester. 0 to # disable. # CLI flag: -ingester.max-series-per-metric [max_series_per_metric: \u003cint\u003e | default = 50000] # The maximum number of active series per user, across the cluster before # replication. 0 to disable. Supported only if -distributor.shard-by-all-labels # is true. # CLI flag: -ingester.max-global-series-per-user [max_global_series_per_user: \u003cint\u003e | default = 0] # The maximum number of active series per metric name, across the cluster before # replication. 0 to disable. # CLI flag: -ingester.max-global-series-per-metric [max_global_series_per_metric: \u003cint\u003e | default = 0] # [Experimental] Enable limits per LabelSet. Supported limits per labelSet: # [max_series] [limits_per_label_set: \u003clist of LimitsPerLabelSet\u003e | default = []] # The maximum number of active metrics with metadata per user, per ingester. 0 # to disable. # CLI flag: -ingester.max-metadata-per-user [max_metadata_per_user: \u003cint\u003e | default = 8000] # The maximum number of metadata per metric, per ingester. 0 to disable. # CLI flag: -ingester.max-metadata-per-metric [max_metadata_per_metric: \u003cint\u003e | default = 10] # The maximum number of active metrics with metadata per user, across the # cluster. 0 to disable. Supported only if -distributor.shard-by-all-labels is # true. # CLI flag: -ingester.max-global-metadata-per-user [max_global_metadata_per_user: \u003cint\u003e | default = 0] # The maximum number of metadata per metric, across the cluster. 0 to disable. # CLI flag: -ingester.max-global-metadata-per-metric [max_global_metadata_per_metric: \u003cint\u003e | default = 0] # [Experimental] Configures the allowed time window for ingestion of # out-of-order samples. Disabled (0s) by default. # CLI flag: -ingester.out-of-order-time-window [out_of_order_time_window: \u003cduration\u003e | default = 0s] # Enables support for exemplars in TSDB and sets the maximum number that will be # stored. less than zero means disabled. If the value is set to zero, cortex # will fallback to blocks-storage.tsdb.max-exemplars value. # CLI flag: -ingester.max-exemplars [max_exemplars: \u003cint\u003e | default = 0] # Maximum number of chunks that can be fetched in a single query from ingesters # and long-term storage. This limit is enforced in the querier, ruler and # store-gateway. 0 to disable. # CLI flag: -querier.max-fetched-chunks-per-query [max_fetched_chunks_per_query: \u003cint\u003e | default = 2000000] # The maximum number of unique series for which a query can fetch samples from # each ingesters and blocks storage. This limit is enforced in the querier, # ruler and store-gateway. 0 to disable # CLI flag: -querier.max-fetched-series-per-query [max_fetched_series_per_query: \u003cint\u003e | default = 0] # Deprecated (use max-fetched-data-bytes-per-query instead): The maximum size of # all chunks in bytes that a query can fetch from each ingester and storage. # This limit is enforced in the querier, ruler and store-gateway. 0 to disable. # CLI flag: -querier.max-fetched-chunk-bytes-per-query [max_fetched_chunk_bytes_per_query: \u003cint\u003e | default = 0] # The maximum combined size of all data that a query can fetch from each # ingester and storage. This limit is enforced in the querier and ruler for # `query`, `query_range` and `series` APIs. 0 to disable. # CLI flag: -querier.max-fetched-data-bytes-per-query [max_fetched_data_bytes_per_query: \u003cint\u003e | default = 0] # Limit how long back data (series and metadata) can be queried, up until # \u003clookback\u003e duration ago. This limit is enforced in the query-frontend, querier # and ruler. If the requested time range is outside the allowed range, the # request will not fail but will be manipulated to only query data within the # allowed time range. 0 to disable. # CLI flag: -querier.max-query-lookback [max_query_lookback: \u003cduration\u003e | default = 0s] # Limit the query time range (end - start time of range query parameter and max # - min of data fetched time range). This limit is enforced in the # query-frontend and ruler (on the received query). 0 to disable. # CLI flag: -store.max-query-length [max_query_length: \u003cduration\u003e | default = 0s] # Maximum number of split queries will be scheduled in parallel by the frontend. # CLI flag: -querier.max-query-parallelism [max_query_parallelism: \u003cint\u003e | default = 14] # Most recent allowed cacheable result per-tenant, to prevent caching very # recent results that might still be in flux. # CLI flag: -frontend.max-cache-freshness [max_cache_freshness: \u003cduration\u003e | default = 1m] # Maximum number of queriers that can handle requests for a single tenant. If # set to 0 or value higher than number of available queriers, *all* queriers # will handle requests for the tenant. If the value is \u003c 1, it will be treated # as a percentage and the gets a percentage of the total queriers. Each frontend # (or query-scheduler, if used) will select the same set of queriers for the # same tenant (given that all queriers are connected to all frontends / # query-schedulers). This option only works with queriers connecting to the # query-frontend / query-scheduler, not when using downstream URL. # CLI flag: -frontend.max-queriers-per-tenant [max_queriers_per_tenant: \u003cfloat\u003e | default = 0] # Maximum number of outstanding requests per tenant per request queue (either # query frontend or query scheduler); requests beyond this error with HTTP 429. # CLI flag: -frontend.max-outstanding-requests-per-tenant [max_outstanding_requests_per_tenant: \u003cint\u003e | default = 100] # Configuration for query priority. query_priority: # Whether queries are assigned with priorities. # CLI flag: -frontend.query-priority.enabled [enabled: \u003cboolean\u003e | default = false] # Priority assigned to all queries by default. Must be a unique value. Use # this as a baseline to make certain queries higher/lower priority. # CLI flag: -frontend.query-priority.default-priority [default_priority: \u003cint\u003e | default = 0] # List of priority definitions. [priorities: \u003clist of PriorityDef\u003e | default = []] # Configuration for query rejection. query_rejection: # Whether query rejection is enabled. # CLI flag: -frontend.query-rejection.enabled [enabled: \u003cboolean\u003e | default = false] # List of query_attributes to match and reject queries. A query is rejected if # it matches any query_attribute in this list. Each query_attribute has # several properties (e.g., regex, time_window, user_agent), and all specified # properties must match for a query_attribute to be considered a match. Only # the specified properties are checked, and an AND operator is applied to # them. [query_attributes: \u003clist of QueryAttribute\u003e | default = []] # Deprecated(use ruler.query-offset instead) and will be removed in v1.19.0: # Duration to delay the evaluation of rules to ensure the underlying metrics # have been pushed to Cortex. # CLI flag: -ruler.evaluation-delay-duration [ruler_evaluation_delay_duration: \u003cduration\u003e | default = 0s] # The default tenant's shard size when the shuffle-sharding strategy is used by # ruler. When this setting is specified in the per-tenant overrides, a value of # 0 disables shuffle sharding for the tenant. # CLI flag: -ruler.tenant-shard-size [ruler_tenant_shard_size: \u003cint\u003e | default = 0] # Maximum number of rules per rule group per-tenant. 0 to disable. # CLI flag: -ruler.max-rules-per-rule-group [ruler_max_rules_per_rule_group: \u003cint\u003e | default = 0] # Maximum number of rule groups per-tenant. 0 to disable. # CLI flag: -ruler.max-rule-groups-per-tenant [ruler_max_rule_groups_per_tenant: \u003cint\u003e | default = 0] # Duration to offset all rule evaluation queries per-tenant. # CLI flag: -ruler.query-offset [ruler_query_offset: \u003cduration\u003e | default = 0s] # external labels for alerting rules [ruler_external_labels: \u003cmap of string (labelName) to string (labelValue)\u003e | default = []] # The default tenant's shard size when the shuffle-sharding strategy is used. # Must be set when the store-gateway sharding is enabled with the # shuffle-sharding strategy. When this setting is specified in the per-tenant # overrides, a value of 0 disables shuffle sharding for the tenant. If the value # is \u003c 1 the shard size will be a percentage of the total store-gateways. # CLI flag: -store-gateway.tenant-shard-size [store_gateway_tenant_shard_size: \u003cfloat\u003e | default = 0] # The maximum number of data bytes to download per gRPC request in Store # Gateway, including Series/LabelNames/LabelValues requests. 0 to disable. # CLI flag: -store-gateway.max-downloaded-bytes-per-request [max_downloaded_bytes_per_request: \u003cint\u003e | default = 0] # Delete blocks containing samples older than the specified retention period. 0 # to disable. # CLI flag: -compactor.blocks-retention-period [compactor_blocks_retention_period: \u003cduration\u003e | default = 0s] # The default tenant's shard size when the shuffle-sharding strategy is used by # the compactor. When this setting is specified in the per-tenant overrides, a # value of 0 disables shuffle sharding for the tenant. # CLI flag: -compactor.tenant-shard-size [compactor_tenant_shard_size: \u003cint\u003e | default = 0] # S3 server-side encryption type. Required to enable server-side encryption # overrides for a specific tenant. If not set, the default S3 client settings # are used. [s3_sse_type: \u003cstring\u003e | default = \"\"] # S3 server-side encryption KMS Key ID. Ignored if the SSE type override is not # set. [s3_sse_kms_key_id: \u003cstring\u003e | default = \"\"] # S3 server-side encryption KMS encryption context. If unset and the key ID # override is set, the encryption context will not be provided to S3. Ignored if # the SSE type override is not set. [s3_sse_kms_encryption_context: \u003cstring\u003e | default = \"\"] # Comma-separated list of network CIDRs to block in Alertmanager receiver # integrations. # CLI flag: -alertmanager.receivers-firewall-block-cidr-networks [alertmanager_receivers_firewall_block_cidr_networks: \u003cstring\u003e | default = \"\"] # True to block private and local addresses in Alertmanager receiver # integrations. It blocks private addresses defined by RFC 1918 (IPv4 # addresses) and RFC 4193 (IPv6 addresses), as well as loopback, local unicast # and local multicast addresses. # CLI flag: -alertmanager.receivers-firewall-block-private-addresses [alertmanager_receivers_firewall_block_private_addresses: \u003cboolean\u003e | default = false] # Per-user rate limit for sending notifications from Alertmanager in # notifications/sec. 0 = rate limit disabled. Negative value = no notifications # are allowed. # CLI flag: -alertmanager.notification-rate-limit [alertmanager_notification_rate_limit: \u003cfloat\u003e | default = 0] # Per-integration notification rate limits. Value is a map, where each key is # integration name and value is a rate-limit (float). On command line, this map # is given in JSON format. Rate limit has the same meaning as # -alertmanager.notification-rate-limit, but only applies for specific # integration. Allowed integration names: webhook, email, pagerduty, opsgenie, # wechat, slack, victorops, pushover, sns, telegram, discord, webex, msteams. # CLI flag: -alertmanager.notification-rate-limit-per-integration [alertmanager_notification_rate_limit_per_integration: \u003cmap of string to float64\u003e | default = {}] # Maximum size of configuration file for Alertmanager that tenant can upload via # Alertmanager API. 0 = no limit. # CLI flag: -alertmanager.max-config-size-bytes [alertmanager_max_config_size_bytes: \u003cint\u003e | default = 0] # Maximum number of templates in tenant's Alertmanager configuration uploaded # via Alertmanager API. 0 = no limit. # CLI flag: -alertmanager.max-templates-count [alertmanager_max_templates_count: \u003cint\u003e | default = 0] # Maximum size of single template in tenant's Alertmanager configuration # uploaded via Alertmanager API. 0 = no limit. # CLI flag: -alertmanager.max-template-size-bytes [alertmanager_max_template_size_bytes: \u003cint\u003e | default = 0] # Maximum number of aggregation groups in Alertmanager's dispatcher that a # tenant can have. Each active aggregation group uses single goroutine. When the # limit is reached, dispatcher will not dispatch alerts that belong to # additional aggregation groups, but existing groups will keep working properly. # 0 = no limit. # CLI flag: -alertmanager.max-dispatcher-aggregation-groups [alertmanager_max_dispatcher_aggregation_groups: \u003cint\u003e | default = 0] # Maximum number of alerts that a single user can have. Inserting more alerts # will fail with a log message and metric increment. 0 = no limit. # CLI flag: -alertmanager.max-alerts-count [alertmanager_max_alerts_count: \u003cint\u003e | default = 0] # Maximum total size of alerts that a single user can have, alert size is the # sum of the bytes of its labels, annotations and generatorURL. Inserting more # alerts will fail with a log message and metric increment. 0 = no limit. # CLI flag: -alertmanager.max-alerts-size-bytes [alertmanager_max_alerts_size_bytes: \u003cint\u003e | default = 0] # list of rule groups to disable [disabled_rule_groups: \u003clist of DisabledRuleGroup\u003e | default = []] memberlist_config The memberlist_config configures the Gossip memberlist.\n# Name of the node in memberlist cluster. Defaults to hostname. # CLI flag: -memberlist.nodename [node_name: \u003cstring\u003e | default = \"\"] # Add random suffix to the node name. # CLI flag: -memberlist.randomize-node-name [randomize_node_name: \u003cboolean\u003e | default = true] # The timeout for establishing a connection with a remote node, and for # read/write operations. # CLI flag: -memberlist.stream-timeout [stream_timeout: \u003cduration\u003e | default = 10s] # Multiplication factor used when sending out messages (factor * log(N+1)). # CLI flag: -memberlist.retransmit-factor [retransmit_factor: \u003cint\u003e | default = 4] # How often to use pull/push sync. # CLI flag: -memberlist.pullpush-interval [pull_push_interval: \u003cduration\u003e | default = 30s] # How often to gossip. # CLI flag: -memberlist.gossip-interval [gossip_interval: \u003cduration\u003e | default = 200ms] # How many nodes to gossip to. # CLI flag: -memberlist.gossip-nodes [gossip_nodes: \u003cint\u003e | default = 3] # How long to keep gossiping to dead nodes, to give them chance to refute their # death. # CLI flag: -memberlist.gossip-to-dead-nodes-time [gossip_to_dead_nodes_time: \u003cduration\u003e | default = 30s] # How soon can dead node's name be reclaimed with new address. 0 to disable. # CLI flag: -memberlist.dead-node-reclaim-time [dead_node_reclaim_time: \u003cduration\u003e | default = 0s] # Enable message compression. This can be used to reduce bandwidth usage at the # cost of slightly more CPU utilization. # CLI flag: -memberlist.compression-enabled [compression_enabled: \u003cboolean\u003e | default = true] # Gossip address to advertise to other members in the cluster. Used for NAT # traversal. # CLI flag: -memberlist.advertise-addr [advertise_addr: \u003cstring\u003e | default = \"\"] # Gossip port to advertise to other members in the cluster. Used for NAT # traversal. # CLI flag: -memberlist.advertise-port [advertise_port: \u003cint\u003e | default = 7946] # Other cluster members to join. Can be specified multiple times. It can be an # IP, hostname or an entry specified in the DNS Service Discovery format. # CLI flag: -memberlist.join [join_members: \u003clist of string\u003e | default = []] # Min backoff duration to join other cluster members. # CLI flag: -memberlist.min-join-backoff [min_join_backoff: \u003cduration\u003e | default = 1s] # Max backoff duration to join other cluster members. # CLI flag: -memberlist.max-join-backoff [max_join_backoff: \u003cduration\u003e | default = 1m] # Max number of retries to join other cluster members. # CLI flag: -memberlist.max-join-retries [max_join_retries: \u003cint\u003e | default = 10] # If this node fails to join memberlist cluster, abort. # CLI flag: -memberlist.abort-if-join-fails [abort_if_cluster_join_fails: \u003cboolean\u003e | default = true] # If not 0, how often to rejoin the cluster. Occasional rejoin can help to fix # the cluster split issue, and is harmless otherwise. For example when using # only few components as a seed nodes (via -memberlist.join), then it's # recommended to use rejoin. If -memberlist.join points to dynamic service that # resolves to all gossiping nodes (eg. Kubernetes headless service), then rejoin # is not needed. # CLI flag: -memberlist.rejoin-interval [rejoin_interval: \u003cduration\u003e | default = 0s] # How long to keep LEFT ingesters in the ring. # CLI flag: -memberlist.left-ingesters-timeout [left_ingesters_timeout: \u003cduration\u003e | default = 5m] # Timeout for leaving memberlist cluster. # CLI flag: -memberlist.leave-timeout [leave_timeout: \u003cduration\u003e | default = 5s] # How much space to use for keeping received and sent messages in memory for # troubleshooting (two buffers). 0 to disable. # CLI flag: -memberlist.message-history-buffer-bytes [message_history_buffer_bytes: \u003cint\u003e | default = 0] # IP address to listen on for gossip messages. Multiple addresses may be # specified. Defaults to 0.0.0.0 # CLI flag: -memberlist.bind-addr [bind_addr: \u003clist of string\u003e | default = []] # Port to listen on for gossip messages. # CLI flag: -memberlist.bind-port [bind_port: \u003cint\u003e | default = 7946] # Timeout used when connecting to other nodes to send packet. # CLI flag: -memberlist.packet-dial-timeout [packet_dial_timeout: \u003cduration\u003e | default = 5s] # Timeout for writing 'packet' data. # CLI flag: -memberlist.packet-write-timeout [packet_write_timeout: \u003cduration\u003e | default = 5s] # Enable TLS on the memberlist transport layer. # CLI flag: -memberlist.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -memberlist.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -memberlist.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -memberlist.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -memberlist.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -memberlist.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] memcached_config The memcached_config block configures how data is stored in Memcached (ie. expiration).\n# How long keys stay in the memcache. # CLI flag: -frontend.memcached.expiration [expiration: \u003cduration\u003e | default = 0s] # How many keys to fetch in each batch. # CLI flag: -frontend.memcached.batchsize [batch_size: \u003cint\u003e | default = 1024] # Maximum active requests to memcache. # CLI flag: -frontend.memcached.parallelism [parallelism: \u003cint\u003e | default = 100] memcached_client_config The memcached_client_config configures the client used to connect to Memcached.\n# Hostname for memcached service to use. If empty and if addresses is unset, no # memcached will be used. # CLI flag: -frontend.memcached.hostname [host: \u003cstring\u003e | default = \"\"] # SRV service used to discover memcache servers. # CLI flag: -frontend.memcached.service [service: \u003cstring\u003e | default = \"memcached\"] # EXPERIMENTAL: Comma separated addresses list in DNS Service Discovery format: # https://cortexmetrics.io/docs/configuration/arguments/#dns-service-discovery # CLI flag: -frontend.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # Maximum time to wait before giving up on memcached requests. # CLI flag: -frontend.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # Maximum number of idle connections in pool. # CLI flag: -frontend.memcached.max-idle-conns [max_idle_conns: \u003cint\u003e | default = 16] # The maximum size of an item stored in memcached. Bigger items are not stored. # If set to 0, no maximum size is enforced. # CLI flag: -frontend.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 0] # Period with which to poll DNS for memcache servers. # CLI flag: -frontend.memcached.update-interval [update_interval: \u003cduration\u003e | default = 1m] # Use consistent hashing to distribute to memcache servers. # CLI flag: -frontend.memcached.consistent-hash [consistent_hash: \u003cboolean\u003e | default = true] # Trip circuit-breaker after this number of consecutive dial failures (if zero # then circuit-breaker is disabled). # CLI flag: -frontend.memcached.circuit-breaker-consecutive-failures [circuit_breaker_consecutive_failures: \u003cint\u003e | default = 10] # Duration circuit-breaker remains open after tripping (if zero then 60 seconds # is used). # CLI flag: -frontend.memcached.circuit-breaker-timeout [circuit_breaker_timeout: \u003cduration\u003e | default = 10s] # Reset circuit-breaker counts after this long (if zero then never reset). # CLI flag: -frontend.memcached.circuit-breaker-interval [circuit_breaker_interval: \u003cduration\u003e | default = 10s] querier_config The querier_config configures the Cortex querier.\n# The maximum number of concurrent queries. # CLI flag: -querier.max-concurrent [max_concurrent: \u003cint\u003e | default = 20] # The timeout for a query. # CLI flag: -querier.timeout [timeout: \u003cduration\u003e | default = 2m] # Deprecated (This feature will be always on after v1.18): Use streaming RPCs # for metadata APIs from ingester. # CLI flag: -querier.ingester-metadata-streaming [ingester_metadata_streaming: \u003cboolean\u003e | default = true] # Use LabelNames ingester RPCs with match params. # CLI flag: -querier.ingester-label-names-with-matchers [ingester_label_names_with_matchers: \u003cboolean\u003e | default = false] # Maximum number of samples a single query can load into memory. # CLI flag: -querier.max-samples [max_samples: \u003cint\u003e | default = 50000000] # Maximum lookback beyond which queries are not sent to ingester. 0 means all # queries are sent to ingester. # CLI flag: -querier.query-ingesters-within [query_ingesters_within: \u003cduration\u003e | default = 0s] # Enable returning samples stats per steps in query response. # CLI flag: -querier.per-step-stats-enabled [per_step_stats_enabled: \u003cboolean\u003e | default = false] # Use compression for metrics query API or instant and range query APIs. # Supports 'gzip' and '' (disable compression) # CLI flag: -querier.response-compression [response_compression: \u003cstring\u003e | default = \"gzip\"] # The time after which a metric should be queried from storage and not just # ingesters. 0 means all queries are sent to store. When running the blocks # storage, if this option is enabled, the time range of the query sent to the # store will be manipulated to ensure the query end is not more recent than 'now # - query-store-after'. # CLI flag: -querier.query-store-after [query_store_after: \u003cduration\u003e | default = 0s] # Maximum duration into the future you can query. 0 to disable. # CLI flag: -querier.max-query-into-future [max_query_into_future: \u003cduration\u003e | default = 10m] # The default evaluation interval or step size for subqueries. # CLI flag: -querier.default-evaluation-interval [default_evaluation_interval: \u003cduration\u003e | default = 1m] # Max number of steps allowed for every subquery expression in query. Number of # steps is calculated using subquery range / step. A value \u003e 0 enables it. # CLI flag: -querier.max-subquery-steps [max_subquery_steps: \u003cint\u003e | default = 0] # Active query tracker monitors active queries, and writes them to the file in # given directory. If Cortex discovers any queries in this log during startup, # it will log them to the log file. Setting to empty value disables active query # tracker, which also disables -querier.max-concurrent option. # CLI flag: -querier.active-query-tracker-dir [active_query_tracker_dir: \u003cstring\u003e | default = \"./active-query-tracker\"] # Time since the last sample after which a time series is considered stale and # ignored by expression evaluations. # CLI flag: -querier.lookback-delta [lookback_delta: \u003cduration\u003e | default = 5m] # Comma separated list of store-gateway addresses in DNS Service Discovery # format. This option should be set when using the blocks storage and the # store-gateway sharding is disabled (when enabled, the store-gateway instances # form a ring and addresses are picked from the ring). # CLI flag: -querier.store-gateway-addresses [store_gateway_addresses: \u003cstring\u003e | default = \"\"] store_gateway_client: # Enable TLS for gRPC client connecting to store-gateway. # CLI flag: -querier.store-gateway-client.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -querier.store-gateway-client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -querier.store-gateway-client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -querier.store-gateway-client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -querier.store-gateway-client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -querier.store-gateway-client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy' and '' (disable compression) # CLI flag: -querier.store-gateway-client.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # EXPERIMENTAL: If enabled, gRPC clients perform health checks for each target # and fail the request if the target is marked as unhealthy. healthcheck_config: # The number of consecutive failed health checks required before considering # a target unhealthy. 0 means disabled. # CLI flag: -querier.store-gateway-client.unhealthy-threshold [unhealthy_threshold: \u003cint\u003e | default = 0] # The approximate amount of time between health checks of an individual # target. # CLI flag: -querier.store-gateway-client.interval [interval: \u003cduration\u003e | default = 5s] # The amount of time during which no response from a target means a failed # health check. # CLI flag: -querier.store-gateway-client.timeout [timeout: \u003cduration\u003e | default = 1s] # If enabled, store gateway query stats will be logged using `info` log level. # CLI flag: -querier.store-gateway-query-stats-enabled [store_gateway_query_stats: \u003cboolean\u003e | default = true] # The maximum number of times we attempt fetching missing blocks from different # store-gateways. If no more store-gateways are left (ie. due to lower # replication factor) than we'll end the retries earlier # CLI flag: -querier.store-gateway-consistency-check-max-attempts [store_gateway_consistency_check_max_attempts: \u003cint\u003e | default = 3] # When distributor's sharding strategy is shuffle-sharding and this setting is \u003e # 0, queriers fetch in-memory series from the minimum set of required ingesters, # selecting only ingesters which may have received series since 'now - lookback # period'. The lookback period should be greater or equal than the configured # 'query store after' and 'query ingesters within'. If this setting is 0, # queriers always query all ingesters (ingesters shuffle sharding on read path # is disabled). # CLI flag: -querier.shuffle-sharding-ingesters-lookback-period [shuffle_sharding_ingesters_lookback_period: \u003cduration\u003e | default = 0s] # Experimental. Use Thanos promql engine # https://github.com/thanos-io/promql-engine rather than the Prometheus promql # engine. # CLI flag: -querier.thanos-engine [thanos_engine: \u003cboolean\u003e | default = false] # If enabled, ignore max query length check at Querier select method. Users can # choose to ignore it since the validation can be done before Querier evaluation # like at Query Frontend or Ruler. # CLI flag: -querier.ignore-max-query-length [ignore_max_query_length: \u003cboolean\u003e | default = false] # [Experimental] If true, experimental promQL functions are enabled. # CLI flag: -querier.enable-promql-experimental-functions [enable_promql_experimental_functions: \u003cboolean\u003e | default = false] query_frontend_config The query_frontend_config configures the Cortex query-frontend.\n# Log queries that are slower than the specified duration. Set to 0 to disable. # Set to \u003c 0 to enable on all queries. # CLI flag: -frontend.log-queries-longer-than [log_queries_longer_than: \u003cduration\u003e | default = 0s] # Max body size for downstream prometheus. # CLI flag: -frontend.max-body-size [max_body_size: \u003cint\u003e | default = 10485760] # True to enable query statistics tracking. When enabled, a message with some # statistics is logged for every query. # CLI flag: -frontend.query-stats-enabled [query_stats_enabled: \u003cboolean\u003e | default = false] # If a querier disconnects without sending notification about graceful shutdown, # the query-frontend will keep the querier in the tenant's shard until the # forget delay has passed. This feature is useful to reduce the blast radius # when shuffle-sharding is enabled. # CLI flag: -query-frontend.querier-forget-delay [querier_forget_delay: \u003cduration\u003e | default = 0s] # DNS hostname used for finding query-schedulers. # CLI flag: -frontend.scheduler-address [scheduler_address: \u003cstring\u003e | default = \"\"] # How often to resolve the scheduler-address, in order to look for new # query-scheduler instances. # CLI flag: -frontend.scheduler-dns-lookup-period [scheduler_dns_lookup_period: \u003cduration\u003e | default = 10s] # Number of concurrent workers forwarding queries to single query-scheduler. # CLI flag: -frontend.scheduler-worker-concurrency [scheduler_worker_concurrency: \u003cint\u003e | default = 5] grpc_client_config: # gRPC client max receive message size (bytes). # CLI flag: -frontend.grpc-client-config.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # gRPC client max send message size (bytes). # CLI flag: -frontend.grpc-client-config.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 16777216] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy', 'snappy-block' ,'zstd' and '' (disable compression) # CLI flag: -frontend.grpc-client-config.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # Rate limit for gRPC client; 0 means disabled. # CLI flag: -frontend.grpc-client-config.grpc-client-rate-limit [rate_limit: \u003cfloat\u003e | default = 0] # Rate limit burst for gRPC client. # CLI flag: -frontend.grpc-client-config.grpc-client-rate-limit-burst [rate_limit_burst: \u003cint\u003e | default = 0] # Enable backoff and retry when we hit ratelimits. # CLI flag: -frontend.grpc-client-config.backoff-on-ratelimits [backoff_on_ratelimits: \u003cboolean\u003e | default = false] backoff_config: # Minimum delay when backing off. # CLI flag: -frontend.grpc-client-config.backoff-min-period [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -frontend.grpc-client-config.backoff-max-period [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -frontend.grpc-client-config.backoff-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -frontend.grpc-client-config.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -frontend.grpc-client-config.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -frontend.grpc-client-config.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -frontend.grpc-client-config.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -frontend.grpc-client-config.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -frontend.grpc-client-config.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # When multiple query-schedulers are available, re-enqueue queries that were # rejected due to too many outstanding requests. # CLI flag: -frontend.retry-on-too-many-outstanding-requests [retry_on_too_many_outstanding_requests: \u003cboolean\u003e | default = false] # Name of network interface to read address from. This address is sent to # query-scheduler and querier, which uses it to send the query response back to # query-frontend. # CLI flag: -frontend.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # URL of downstream Prometheus. # CLI flag: -frontend.downstream-url [downstream_url: \u003cstring\u003e | default = \"\"] query_range_config The query_range_config configures the query splitting and caching in the Cortex query-frontend.\n# Split queries by an interval and execute in parallel, 0 disables it. You # should use a multiple of 24 hours (same as the storage bucketing scheme), to # avoid queriers downloading and processing the same chunks. This also # determines how cache keys are chosen when result caching is enabled # CLI flag: -querier.split-queries-by-interval [split_queries_by_interval: \u003cduration\u003e | default = 0s] # Mutate incoming queries to align their start and end with their step. # CLI flag: -querier.align-querier-with-step [align_queries_with_step: \u003cboolean\u003e | default = false] results_cache: cache: # Enable in-memory cache. # CLI flag: -frontend.cache.enable-fifocache [enable_fifocache: \u003cboolean\u003e | default = false] # The default validity of entries for caches unless overridden. # CLI flag: -frontend.default-validity [default_validity: \u003cduration\u003e | default = 0s] background: # At what concurrency to write back to cache. # CLI flag: -frontend.background.write-back-concurrency [writeback_goroutines: \u003cint\u003e | default = 10] # How many key batches to buffer for background write-back. # CLI flag: -frontend.background.write-back-buffer [writeback_buffer: \u003cint\u003e | default = 10000] # The memcached_config block configures how data is stored in Memcached (ie. # expiration). [memcached: \u003cmemcached_config\u003e] # The memcached_client_config configures the client used to connect to # Memcached. [memcached_client: \u003cmemcached_client_config\u003e] # The redis_config configures the Redis backend cache. [redis: \u003credis_config\u003e] # The fifo_cache_config configures the local in-memory cache. [fifocache: \u003cfifo_cache_config\u003e] # Use compression in results cache. Supported values are: 'snappy' and '' # (disable compression). # CLI flag: -frontend.compression [compression: \u003cstring\u003e | default = \"\"] # Cache Statistics queryable samples on results cache. # CLI flag: -frontend.cache-queryable-samples-stats [cache_queryable_samples_stats: \u003cboolean\u003e | default = false] # Cache query results. # CLI flag: -querier.cache-results [cache_results: \u003cboolean\u003e | default = false] # Maximum number of retries for a single request; beyond this, the downstream # error is returned. # CLI flag: -querier.max-retries-per-request [max_retries: \u003cint\u003e | default = 5] # List of headers forwarded by the query Frontend to downstream querier. # CLI flag: -frontend.forward-headers-list [forward_headers_list: \u003clist of string\u003e | default = []] redis_config The redis_config configures the Redis backend cache.\n# Redis Server endpoint to use for caching. A comma-separated list of endpoints # for Redis Cluster or Redis Sentinel. If empty, no redis will be used. # CLI flag: -frontend.redis.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # Redis Sentinel master name. An empty string for Redis Server or Redis Cluster. # CLI flag: -frontend.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # Maximum time to wait before giving up on redis requests. # CLI flag: -frontend.redis.timeout [timeout: \u003cduration\u003e | default = 500ms] # How long keys stay in the redis. # CLI flag: -frontend.redis.expiration [expiration: \u003cduration\u003e | default = 0s] # Database index. # CLI flag: -frontend.redis.db [db: \u003cint\u003e | default = 0] # Maximum number of connections in the pool. # CLI flag: -frontend.redis.pool-size [pool_size: \u003cint\u003e | default = 0] # Password to use when connecting to redis. # CLI flag: -frontend.redis.password [password: \u003cstring\u003e | default = \"\"] # Enable connecting to redis with TLS. # CLI flag: -frontend.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Skip validating server certificate. # CLI flag: -frontend.redis.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # Close connections after remaining idle for this duration. If the value is # zero, then idle connections are not closed. # CLI flag: -frontend.redis.idle-timeout [idle_timeout: \u003cduration\u003e | default = 0s] # Close connections older than this duration. If the value is zero, then the # pool does not close connections based on age. # CLI flag: -frontend.redis.max-connection-age [max_connection_age: \u003cduration\u003e | default = 0s] ruler_config The ruler_config configures the Cortex ruler.\n# [Experimental] GRPC listen address of the Query Frontend, in host:port format. # If set, Ruler queries to Query Frontends via gRPC. If not set, ruler queries # to Ingesters directly. # CLI flag: -ruler.frontend-address [frontend_address: \u003cstring\u003e | default = \"\"] frontend_client: # gRPC client max receive message size (bytes). # CLI flag: -ruler.frontendClient.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # gRPC client max send message size (bytes). # CLI flag: -ruler.frontendClient.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 16777216] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy', 'snappy-block' ,'zstd' and '' (disable compression) # CLI flag: -ruler.frontendClient.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # Rate limit for gRPC client; 0 means disabled. # CLI flag: -ruler.frontendClient.grpc-client-rate-limit [rate_limit: \u003cfloat\u003e | default = 0] # Rate limit burst for gRPC client. # CLI flag: -ruler.frontendClient.grpc-client-rate-limit-burst [rate_limit_burst: \u003cint\u003e | default = 0] # Enable backoff and retry when we hit ratelimits. # CLI flag: -ruler.frontendClient.backoff-on-ratelimits [backoff_on_ratelimits: \u003cboolean\u003e | default = false] backoff_config: # Minimum delay when backing off. # CLI flag: -ruler.frontendClient.backoff-min-period [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -ruler.frontendClient.backoff-max-period [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -ruler.frontendClient.backoff-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -ruler.frontendClient.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -ruler.frontendClient.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -ruler.frontendClient.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -ruler.frontendClient.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -ruler.frontendClient.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -ruler.frontendClient.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # URL of alerts return path. # CLI flag: -ruler.external.url [external_url: \u003curl\u003e | default = ] # Labels to add to all alerts. [external_labels: \u003cmap of string (labelName) to string (labelValue)\u003e | default = []] ruler_client: # gRPC client max receive message size (bytes). # CLI flag: -ruler.client.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # gRPC client max send message size (bytes). # CLI flag: -ruler.client.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 16777216] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy', 'snappy-block' ,'zstd' and '' (disable compression) # CLI flag: -ruler.client.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # Rate limit for gRPC client; 0 means disabled. # CLI flag: -ruler.client.grpc-client-rate-limit [rate_limit: \u003cfloat\u003e | default = 0] # Rate limit burst for gRPC client. # CLI flag: -ruler.client.grpc-client-rate-limit-burst [rate_limit_burst: \u003cint\u003e | default = 0] # Enable backoff and retry when we hit ratelimits. # CLI flag: -ruler.client.backoff-on-ratelimits [backoff_on_ratelimits: \u003cboolean\u003e | default = false] backoff_config: # Minimum delay when backing off. # CLI flag: -ruler.client.backoff-min-period [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -ruler.client.backoff-max-period [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -ruler.client.backoff-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -ruler.client.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -ruler.client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -ruler.client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -ruler.client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -ruler.client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -ruler.client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # Timeout for downstream rulers. # CLI flag: -ruler.client.remote-timeout [remote_timeout: \u003cduration\u003e | default = 2m] # How frequently to evaluate rules # CLI flag: -ruler.evaluation-interval [evaluation_interval: \u003cduration\u003e | default = 1m] # How frequently to poll for rule changes # CLI flag: -ruler.poll-interval [poll_interval: \u003cduration\u003e | default = 1m] # file path to store temporary rule files for the prometheus rule managers # CLI flag: -ruler.rule-path [rule_path: \u003cstring\u003e | default = \"/rules\"] # Comma-separated list of URL(s) of the Alertmanager(s) to send notifications # to. Each Alertmanager URL is treated as a separate group in the configuration. # Multiple Alertmanagers in HA per group can be supported by using DNS # resolution via -ruler.alertmanager-discovery. # CLI flag: -ruler.alertmanager-url [alertmanager_url: \u003cstring\u003e | default = \"\"] # Use DNS SRV records to discover Alertmanager hosts. # CLI flag: -ruler.alertmanager-discovery [enable_alertmanager_discovery: \u003cboolean\u003e | default = false] # How long to wait between refreshing DNS resolutions of Alertmanager hosts. # CLI flag: -ruler.alertmanager-refresh-interval [alertmanager_refresh_interval: \u003cduration\u003e | default = 1m] # Capacity of the queue for notifications to be sent to the Alertmanager. # CLI flag: -ruler.notification-queue-capacity [notification_queue_capacity: \u003cint\u003e | default = 10000] # HTTP timeout duration when sending notifications to the Alertmanager. # CLI flag: -ruler.notification-timeout [notification_timeout: \u003cduration\u003e | default = 10s] alertmanager_client: # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -ruler.alertmanager-client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -ruler.alertmanager-client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -ruler.alertmanager-client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -ruler.alertmanager-client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -ruler.alertmanager-client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # HTTP Basic authentication username. It overrides the username set in the URL # (if any). # CLI flag: -ruler.alertmanager-client.basic-auth-username [basic_auth_username: \u003cstring\u003e | default = \"\"] # HTTP Basic authentication password. It overrides the password set in the URL # (if any). # CLI flag: -ruler.alertmanager-client.basic-auth-password [basic_auth_password: \u003cstring\u003e | default = \"\"] # Max time to tolerate outage for restoring \"for\" state of alert. # CLI flag: -ruler.for-outage-tolerance [for_outage_tolerance: \u003cduration\u003e | default = 1h] # Minimum duration between alert and restored \"for\" state. This is maintained # only for alerts with configured \"for\" time greater than grace period. # CLI flag: -ruler.for-grace-period [for_grace_period: \u003cduration\u003e | default = 10m] # Minimum amount of time to wait before resending an alert to Alertmanager. # CLI flag: -ruler.resend-delay [resend_delay: \u003cduration\u003e | default = 1m] # If enabled, rules from a single rule group can be evaluated concurrently if # there is no dependency between each other. Max concurrency for each rule group # is controlled via ruler.max-concurrent-evals flag. # CLI flag: -ruler.concurrent-evals-enabled [concurrent_evals_enabled: \u003cboolean\u003e | default = false] # Max concurrency for a single rule group to evaluate independent rules. # CLI flag: -ruler.max-concurrent-evals [max_concurrent_evals: \u003cint\u003e | default = 1] # Distribute rule evaluation using ring backend # CLI flag: -ruler.enable-sharding [enable_sharding: \u003cboolean\u003e | default = false] # The sharding strategy to use. Supported values are: default, shuffle-sharding. # CLI flag: -ruler.sharding-strategy [sharding_strategy: \u003cstring\u003e | default = \"default\"] # Time to spend searching for a pending ruler when shutting down. # CLI flag: -ruler.search-pending-for [search_pending_for: \u003cduration\u003e | default = 5m] ring: kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -ruler.ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -ruler.ring.prefix [prefix: \u003cstring\u003e | default = \"rulers/\"] dynamodb: # Region to access dynamodb. # CLI flag: -ruler.ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -ruler.ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -ruler.ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -ruler.ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -ruler.ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: ruler.ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: ruler.ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -ruler.ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -ruler.ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -ruler.ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -ruler.ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -ruler.ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 5s] # The heartbeat timeout after which rulers are considered unhealthy within the # ring. 0 = never (timeout disabled). # CLI flag: -ruler.ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # EXPERIMENTAL: The replication factor to use when loading rule groups for API # HA. # CLI flag: -ruler.ring.replication-factor [replication_factor: \u003cint\u003e | default = 1] # EXPERIMENTAL: True to enable zone-awareness and load rule groups across # different availability zones for API HA. # CLI flag: -ruler.ring.zone-awareness-enabled [zone_awareness_enabled: \u003cboolean\u003e | default = false] # EXPERIMENTAL: File path where tokens are stored. If empty, tokens are not # stored at shutdown and restored at startup. # CLI flag: -ruler.ring.tokens-file-path [tokens_file_path: \u003cstring\u003e | default = \"\"] # Name of network interface to read address from. # CLI flag: -ruler.ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # Number of tokens for each ruler. # CLI flag: -ruler.ring.num-tokens [num_tokens: \u003cint\u003e | default = 128] # The sleep seconds when ruler is shutting down. Need to be close to or larger # than KV Store information propagation delay # CLI flag: -ruler.ring.final-sleep [final_sleep: \u003cduration\u003e | default = 0s] # Keep instance in the ring on shut down. # CLI flag: -ruler.ring.keep-instance-in-the-ring-on-shutdown [keep_instance_in_the_ring_on_shutdown: \u003cboolean\u003e | default = false] # Period with which to attempt to flush rule groups. # CLI flag: -ruler.flush-period [flush_period: \u003cduration\u003e | default = 1m] # Enable the ruler api # CLI flag: -experimental.ruler.enable-api [enable_api: \u003cboolean\u003e | default = false] # EXPERIMENTAL: Remove duplicate rules in the prometheus rules and alerts API # response. If there are duplicate rules the rule with the latest evaluation # timestamp will be kept. # CLI flag: -experimental.ruler.api-deduplicate-rules [api_deduplicate_rules: \u003cboolean\u003e | default = false] # Comma separated list of tenants whose rules this ruler can evaluate. If # specified, only these tenants will be handled by ruler, otherwise this ruler # can process rules from all tenants. Subject to sharding. # CLI flag: -ruler.enabled-tenants [enabled_tenants: \u003cstring\u003e | default = \"\"] # Comma separated list of tenants whose rules this ruler cannot evaluate. If # specified, a ruler that would normally pick the specified tenant(s) for # processing will ignore them instead. Subject to sharding. # CLI flag: -ruler.disabled-tenants [disabled_tenants: \u003cstring\u003e | default = \"\"] # Report query statistics for ruler queries to complete as a per user metric and # as an info level log message. # CLI flag: -ruler.query-stats-enabled [query_stats_enabled: \u003cboolean\u003e | default = false] # Disable the rule_group label on exported metrics # CLI flag: -ruler.disable-rule-group-label [disable_rule_group_label: \u003cboolean\u003e | default = false] # Enable high availability # CLI flag: -ruler.enable-ha-evaluation [enable_ha_evaluation: \u003cboolean\u003e | default = false] # Timeout duration for non-primary rulers during liveness checks. If the check # times out, the non-primary ruler will evaluate the rule group. Applicable when # ruler.enable-ha-evaluation is true. # CLI flag: -ruler.liveness-check-timeout [liveness_check_timeout: \u003cduration\u003e | default = 1s] ruler_storage_config The ruler_storage_config configures the Cortex ruler storage backend.\n# Backend storage to use. Supported backends are: s3, gcs, azure, swift, # filesystem, configdb, local. # CLI flag: -ruler-storage.backend [backend: \u003cstring\u003e | default = \"s3\"] s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an # S3-compatible service in hostname:port format. # CLI flag: -ruler-storage.s3.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # S3 region. If unset, the client will issue a S3 GetBucketLocation API call # to autodetect it. # CLI flag: -ruler-storage.s3.region [region: \u003cstring\u003e | default = \"\"] # S3 bucket name # CLI flag: -ruler-storage.s3.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # S3 secret access key # CLI flag: -ruler-storage.s3.secret-access-key [secret_access_key: \u003cstring\u003e | default = \"\"] # S3 access key ID # CLI flag: -ruler-storage.s3.access-key-id [access_key_id: \u003cstring\u003e | default = \"\"] # If enabled, use http:// for the S3 endpoint instead of https://. This could # be useful in local dev/test environments while using an S3-compatible # backend storage, like Minio. # CLI flag: -ruler-storage.s3.insecure [insecure: \u003cboolean\u003e | default = false] # The signature version to use for authenticating against S3. Supported values # are: v4, v2. # CLI flag: -ruler-storage.s3.signature-version [signature_version: \u003cstring\u003e | default = \"v4\"] # The s3 bucket lookup style. Supported values are: auto, virtual-hosted, # path. # CLI flag: -ruler-storage.s3.bucket-lookup-type [bucket_lookup_type: \u003cstring\u003e | default = \"auto\"] # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum # algorithm to verify the provided digest. If false, use CRC32C algorithm # instead. # CLI flag: -ruler-storage.s3.send-content-md5 [send_content_md5: \u003cboolean\u003e | default = true] # The list api version. Supported values are: v1, v2, and ''. # CLI flag: -ruler-storage.s3.list-objects-version [list_objects_version: \u003cstring\u003e | default = \"\"] # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: ruler-storage [sse: \u003cs3_sse_config\u003e] http: # The time an idle connection will remain idle before closing. # CLI flag: -ruler-storage.s3.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -ruler-storage.s3.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -ruler-storage.s3.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -ruler-storage.s3.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -ruler-storage.s3.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -ruler-storage.s3.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -ruler-storage.s3.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -ruler-storage.s3.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] gcs: # GCS bucket name # CLI flag: -ruler-storage.gcs.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # JSON representing either a Google Developers Console client_credentials.json # file or a Google Developers service account key file. If empty, fallback to # Google default logic. # CLI flag: -ruler-storage.gcs.service-account [service_account: \u003cstring\u003e | default = \"\"] azure: # Azure storage account name # CLI flag: -ruler-storage.azure.account-name [account_name: \u003cstring\u003e | default = \"\"] # Azure storage account key # CLI flag: -ruler-storage.azure.account-key [account_key: \u003cstring\u003e | default = \"\"] # The values of `account-name` and `endpoint-suffix` values will not be # ignored if `connection-string` is set. Use this method over `account-key` if # you need to authenticate via a SAS token or if you use the Azurite emulator. # CLI flag: -ruler-storage.azure.connection-string [connection_string: \u003cstring\u003e | default = \"\"] # Azure storage container name # CLI flag: -ruler-storage.azure.container-name [container_name: \u003cstring\u003e | default = \"\"] # Azure storage endpoint suffix without schema. The account name will be # prefixed to this value to create the FQDN # CLI flag: -ruler-storage.azure.endpoint-suffix [endpoint_suffix: \u003cstring\u003e | default = \"\"] # Number of retries for recoverable errors # CLI flag: -ruler-storage.azure.max-retries [max_retries: \u003cint\u003e | default = 20] # Deprecated: Azure storage MSI resource. It will be set automatically by # Azure SDK. # CLI flag: -ruler-storage.azure.msi-resource [msi_resource: \u003cstring\u003e | default = \"\"] # Azure storage MSI resource managed identity client Id. If not supplied # default Azure credential will be used. Set it to empty if you need to # authenticate via Azure Workload Identity. # CLI flag: -ruler-storage.azure.user-assigned-id [user_assigned_id: \u003cstring\u003e | default = \"\"] http: # The time an idle connection will remain idle before closing. # CLI flag: -ruler-storage.azure.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -ruler-storage.azure.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -ruler-storage.azure.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -ruler-storage.azure.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -ruler-storage.azure.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -ruler-storage.azure.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -ruler-storage.azure.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -ruler-storage.azure.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] swift: # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -ruler-storage.swift.auth-version [auth_version: \u003cint\u003e | default = 0] # OpenStack Swift authentication URL # CLI flag: -ruler-storage.swift.auth-url [auth_url: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential ID. # CLI flag: -ruler-storage.swift.application-credential-id [application_credential_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential name. # CLI flag: -ruler-storage.swift.application-credential-name [application_credential_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential secret. # CLI flag: -ruler-storage.swift.application-credential-secret [application_credential_secret: \u003cstring\u003e | default = \"\"] # OpenStack Swift username. # CLI flag: -ruler-storage.swift.username [username: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -ruler-storage.swift.user-domain-name [user_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -ruler-storage.swift.user-domain-id [user_domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user ID. # CLI flag: -ruler-storage.swift.user-id [user_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift API key. # CLI flag: -ruler-storage.swift.password [password: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -ruler-storage.swift.domain-id [domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -ruler-storage.swift.domain-name [domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift project ID (v2,v3 auth only). # CLI flag: -ruler-storage.swift.project-id [project_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift project name (v2,v3 auth only). # CLI flag: -ruler-storage.swift.project-name [project_name: \u003cstring\u003e | default = \"\"] # ID of the OpenStack Swift project's domain (v3 auth only), only needed if it # differs the from user domain. # CLI flag: -ruler-storage.swift.project-domain-id [project_domain_id: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift project's domain (v3 auth only), only needed if # it differs from the user domain. # CLI flag: -ruler-storage.swift.project-domain-name [project_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift Region to use (v2,v3 auth only). # CLI flag: -ruler-storage.swift.region-name [region_name: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift container to put chunks in. # CLI flag: -ruler-storage.swift.container-name [container_name: \u003cstring\u003e | default = \"\"] # Max retries on requests error. # CLI flag: -ruler-storage.swift.max-retries [max_retries: \u003cint\u003e | default = 3] # Time after which a connection attempt is aborted. # CLI flag: -ruler-storage.swift.connect-timeout [connect_timeout: \u003cduration\u003e | default = 10s] # Time after which an idle request is aborted. The timeout watchdog is reset # each time some data is received, so the timeout triggers after X time no # data is received on a request. # CLI flag: -ruler-storage.swift.request-timeout [request_timeout: \u003cduration\u003e | default = 5s] filesystem: # Local filesystem storage directory. # CLI flag: -ruler-storage.filesystem.dir [dir: \u003cstring\u003e | default = \"\"] # The configstore_config configures the config database storing rules and # alerts, and is used by the Cortex alertmanager. # The CLI flags prefix for this block config is: ruler-storage [configdb: \u003cconfigstore_config\u003e] local: # Directory to scan for rules # CLI flag: -ruler-storage.local.directory [directory: \u003cstring\u003e | default = \"\"] runtime_configuration_storage_config The runtime_configuration_storage_config configures the storage backend for the runtime configuration file.\n# How often to check runtime config file. # CLI flag: -runtime-config.reload-period [period: \u003cduration\u003e | default = 10s] # File with the configuration that can be updated in runtime. # CLI flag: -runtime-config.file [file: \u003cstring\u003e | default = \"\"] # Backend storage to use. Supported backends are: s3, gcs, azure, swift, # filesystem. # CLI flag: -runtime-config.backend [backend: \u003cstring\u003e | default = \"filesystem\"] s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an # S3-compatible service in hostname:port format. # CLI flag: -runtime-config.s3.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # S3 region. If unset, the client will issue a S3 GetBucketLocation API call # to autodetect it. # CLI flag: -runtime-config.s3.region [region: \u003cstring\u003e | default = \"\"] # S3 bucket name # CLI flag: -runtime-config.s3.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # S3 secret access key # CLI flag: -runtime-config.s3.secret-access-key [secret_access_key: \u003cstring\u003e | default = \"\"] # S3 access key ID # CLI flag: -runtime-config.s3.access-key-id [access_key_id: \u003cstring\u003e | default = \"\"] # If enabled, use http:// for the S3 endpoint instead of https://. This could # be useful in local dev/test environments while using an S3-compatible # backend storage, like Minio. # CLI flag: -runtime-config.s3.insecure [insecure: \u003cboolean\u003e | default = false] # The signature version to use for authenticating against S3. Supported values # are: v4, v2. # CLI flag: -runtime-config.s3.signature-version [signature_version: \u003cstring\u003e | default = \"v4\"] # The s3 bucket lookup style. Supported values are: auto, virtual-hosted, # path. # CLI flag: -runtime-config.s3.bucket-lookup-type [bucket_lookup_type: \u003cstring\u003e | default = \"auto\"] # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum # algorithm to verify the provided digest. If false, use CRC32C algorithm # instead. # CLI flag: -runtime-config.s3.send-content-md5 [send_content_md5: \u003cboolean\u003e | default = true] # The list api version. Supported values are: v1, v2, and ''. # CLI flag: -runtime-config.s3.list-objects-version [list_objects_version: \u003cstring\u003e | default = \"\"] # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: runtime-config [sse: \u003cs3_sse_config\u003e] http: # The time an idle connection will remain idle before closing. # CLI flag: -runtime-config.s3.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -runtime-config.s3.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -runtime-config.s3.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -runtime-config.s3.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -runtime-config.s3.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -runtime-config.s3.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -runtime-config.s3.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -runtime-config.s3.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] gcs: # GCS bucket name # CLI flag: -runtime-config.gcs.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # JSON representing either a Google Developers Console client_credentials.json # file or a Google Developers service account key file. If empty, fallback to # Google default logic. # CLI flag: -runtime-config.gcs.service-account [service_account: \u003cstring\u003e | default = \"\"] azure: # Azure storage account name # CLI flag: -runtime-config.azure.account-name [account_name: \u003cstring\u003e | default = \"\"] # Azure storage account key # CLI flag: -runtime-config.azure.account-key [account_key: \u003cstring\u003e | default = \"\"] # The values of `account-name` and `endpoint-suffix` values will not be # ignored if `connection-string` is set. Use this method over `account-key` if # you need to authenticate via a SAS token or if you use the Azurite emulator. # CLI flag: -runtime-config.azure.connection-string [connection_string: \u003cstring\u003e | default = \"\"] # Azure storage container name # CLI flag: -runtime-config.azure.container-name [container_name: \u003cstring\u003e | default = \"\"] # Azure storage endpoint suffix without schema. The account name will be # prefixed to this value to create the FQDN # CLI flag: -runtime-config.azure.endpoint-suffix [endpoint_suffix: \u003cstring\u003e | default = \"\"] # Number of retries for recoverable errors # CLI flag: -runtime-config.azure.max-retries [max_retries: \u003cint\u003e | default = 20] # Deprecated: Azure storage MSI resource. It will be set automatically by # Azure SDK. # CLI flag: -runtime-config.azure.msi-resource [msi_resource: \u003cstring\u003e | default = \"\"] # Azure storage MSI resource managed identity client Id. If not supplied # default Azure credential will be used. Set it to empty if you need to # authenticate via Azure Workload Identity. # CLI flag: -runtime-config.azure.user-assigned-id [user_assigned_id: \u003cstring\u003e | default = \"\"] http: # The time an idle connection will remain idle before closing. # CLI flag: -runtime-config.azure.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -runtime-config.azure.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -runtime-config.azure.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -runtime-config.azure.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -runtime-config.azure.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -runtime-config.azure.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -runtime-config.azure.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -runtime-config.azure.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] swift: # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -runtime-config.swift.auth-version [auth_version: \u003cint\u003e | default = 0] # OpenStack Swift authentication URL # CLI flag: -runtime-config.swift.auth-url [auth_url: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential ID. # CLI flag: -runtime-config.swift.application-credential-id [application_credential_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential name. # CLI flag: -runtime-config.swift.application-credential-name [application_credential_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential secret. # CLI flag: -runtime-config.swift.application-credential-secret [application_credential_secret: \u003cstring\u003e | default = \"\"] # OpenStack Swift username. # CLI flag: -runtime-config.swift.username [username: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -runtime-config.swift.user-domain-name [user_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -runtime-config.swift.user-domain-id [user_domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user ID. # CLI flag: -runtime-config.swift.user-id [user_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift API key. # CLI flag: -runtime-config.swift.password [password: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -runtime-config.swift.domain-id [domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -runtime-config.swift.domain-name [domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift project ID (v2,v3 auth only). # CLI flag: -runtime-config.swift.project-id [project_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift project name (v2,v3 auth only). # CLI flag: -runtime-config.swift.project-name [project_name: \u003cstring\u003e | default = \"\"] # ID of the OpenStack Swift project's domain (v3 auth only), only needed if it # differs the from user domain. # CLI flag: -runtime-config.swift.project-domain-id [project_domain_id: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift project's domain (v3 auth only), only needed if # it differs from the user domain. # CLI flag: -runtime-config.swift.project-domain-name [project_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift Region to use (v2,v3 auth only). # CLI flag: -runtime-config.swift.region-name [region_name: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift container to put chunks in. # CLI flag: -runtime-config.swift.container-name [container_name: \u003cstring\u003e | default = \"\"] # Max retries on requests error. # CLI flag: -runtime-config.swift.max-retries [max_retries: \u003cint\u003e | default = 3] # Time after which a connection attempt is aborted. # CLI flag: -runtime-config.swift.connect-timeout [connect_timeout: \u003cduration\u003e | default = 10s] # Time after which an idle request is aborted. The timeout watchdog is reset # each time some data is received, so the timeout triggers after X time no # data is received on a request. # CLI flag: -runtime-config.swift.request-timeout [request_timeout: \u003cduration\u003e | default = 5s] filesystem: # Local filesystem storage directory. # CLI flag: -runtime-config.filesystem.dir [dir: \u003cstring\u003e | default = \"\"] s3_sse_config The s3_sse_config configures the S3 server-side encryption. The supported CLI flags \u003cprefix\u003e used to reference this config block are:\nalertmanager-storage blocks-storage ruler-storage runtime-config # Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3. # CLI flag: -\u003cprefix\u003e.s3.sse.type [type: \u003cstring\u003e | default = \"\"] # KMS Key ID used to encrypt objects in S3 # CLI flag: -\u003cprefix\u003e.s3.sse.kms-key-id [kms_key_id: \u003cstring\u003e | default = \"\"] # KMS Encryption Context used for object encryption. It expects JSON formatted # string. # CLI flag: -\u003cprefix\u003e.s3.sse.kms-encryption-context [kms_encryption_context: \u003cstring\u003e | default = \"\"] server_config The server_config configures the HTTP and gRPC server of the launched service(s).\n# HTTP server listen network, default tcp # CLI flag: -server.http-listen-network [http_listen_network: \u003cstring\u003e | default = \"tcp\"] # HTTP server listen address. # CLI flag: -server.http-listen-address [http_listen_address: \u003cstring\u003e | default = \"\"] # HTTP server listen port. # CLI flag: -server.http-listen-port [http_listen_port: \u003cint\u003e | default = 80] # Maximum number of simultaneous http connections, \u003c=0 to disable # CLI flag: -server.http-conn-limit [http_listen_conn_limit: \u003cint\u003e | default = 0] # gRPC server listen network # CLI flag: -server.grpc-listen-network [grpc_listen_network: \u003cstring\u003e | default = \"tcp\"] # gRPC server listen address. # CLI flag: -server.grpc-listen-address [grpc_listen_address: \u003cstring\u003e | default = \"\"] # gRPC server listen port. # CLI flag: -server.grpc-listen-port [grpc_listen_port: \u003cint\u003e | default = 9095] # Maximum number of simultaneous grpc connections, \u003c=0 to disable # CLI flag: -server.grpc-conn-limit [grpc_listen_conn_limit: \u003cint\u003e | default = 0] # Comma-separated list of cipher suites to use. If blank, the default Go cipher # suites is used. # CLI flag: -server.tls-cipher-suites [tls_cipher_suites: \u003cstring\u003e | default = \"\"] # Minimum TLS version to use. Allowed values: VersionTLS10, VersionTLS11, # VersionTLS12, VersionTLS13. If blank, the Go TLS minimum version is used. # CLI flag: -server.tls-min-version [tls_min_version: \u003cstring\u003e | default = \"\"] http_tls_config: # HTTP server cert path. # CLI flag: -server.http-tls-cert-path [cert_file: \u003cstring\u003e | default = \"\"] # HTTP server key path. # CLI flag: -server.http-tls-key-path [key_file: \u003cstring\u003e | default = \"\"] # HTTP TLS Client Auth type. # CLI flag: -server.http-tls-client-auth [client_auth_type: \u003cstring\u003e | default = \"\"] # HTTP TLS Client CA path. # CLI flag: -server.http-tls-ca-path [client_ca_file: \u003cstring\u003e | default = \"\"] grpc_tls_config: # GRPC TLS server cert path. # CLI flag: -server.grpc-tls-cert-path [cert_file: \u003cstring\u003e | default = \"\"] # GRPC TLS server key path. # CLI flag: -server.grpc-tls-key-path [key_file: \u003cstring\u003e | default = \"\"] # GRPC TLS Client Auth type. # CLI flag: -server.grpc-tls-client-auth [client_auth_type: \u003cstring\u003e | default = \"\"] # GRPC TLS Client CA path. # CLI flag: -server.grpc-tls-ca-path [client_ca_file: \u003cstring\u003e | default = \"\"] # Register the intrumentation handlers (/metrics etc). # CLI flag: -server.register-instrumentation [register_instrumentation: \u003cboolean\u003e | default = true] # Timeout for graceful shutdowns # CLI flag: -server.graceful-shutdown-timeout [graceful_shutdown_timeout: \u003cduration\u003e | default = 30s] # Read timeout for HTTP server # CLI flag: -server.http-read-timeout [http_server_read_timeout: \u003cduration\u003e | default = 30s] # Write timeout for HTTP server # CLI flag: -server.http-write-timeout [http_server_write_timeout: \u003cduration\u003e | default = 30s] # Idle timeout for HTTP server # CLI flag: -server.http-idle-timeout [http_server_idle_timeout: \u003cduration\u003e | default = 2m] # Limit on the size of a gRPC message this server can receive (bytes). # CLI flag: -server.grpc-max-recv-msg-size-bytes [grpc_server_max_recv_msg_size: \u003cint\u003e | default = 4194304] # Limit on the size of a gRPC message this server can send (bytes). # CLI flag: -server.grpc-max-send-msg-size-bytes [grpc_server_max_send_msg_size: \u003cint\u003e | default = 4194304] # Limit on the number of concurrent streams for gRPC calls (0 = unlimited) # CLI flag: -server.grpc-max-concurrent-streams [grpc_server_max_concurrent_streams: \u003cint\u003e | default = 100] # Number of worker goroutines that should be used to process incoming # streams.Setting this 0 (default) will disable workers and spawn a new # goroutine for each stream. # CLI flag: -server.grpc_server-num-stream-workers [grpc_server_num_stream_workers: \u003cint\u003e | default = 0] # The duration after which an idle connection should be closed. Default: # infinity # CLI flag: -server.grpc.keepalive.max-connection-idle [grpc_server_max_connection_idle: \u003cduration\u003e | default = 2562047h47m16.854775807s] # The duration for the maximum amount of time a connection may exist before it # will be closed. Default: infinity # CLI flag: -server.grpc.keepalive.max-connection-age [grpc_server_max_connection_age: \u003cduration\u003e | default = 2562047h47m16.854775807s] # An additive period after max-connection-age after which the connection will be # forcibly closed. Default: infinity # CLI flag: -server.grpc.keepalive.max-connection-age-grace [grpc_server_max_connection_age_grace: \u003cduration\u003e | default = 2562047h47m16.854775807s] # Duration after which a keepalive probe is sent in case of no activity over the # connection., Default: 2h # CLI flag: -server.grpc.keepalive.time [grpc_server_keepalive_time: \u003cduration\u003e | default = 2h] # After having pinged for keepalive check, the duration after which an idle # connection should be closed, Default: 20s # CLI flag: -server.grpc.keepalive.timeout [grpc_server_keepalive_timeout: \u003cduration\u003e | default = 20s] # Minimum amount of time a client should wait before sending a keepalive ping. # If client sends keepalive ping more often, server will send GOAWAY and close # the connection. # CLI flag: -server.grpc.keepalive.min-time-between-pings [grpc_server_min_time_between_pings: \u003cduration\u003e | default = 10s] # If true, server allows keepalive pings even when there are no active # streams(RPCs). If false, and client sends ping when there are no active # streams, server will send GOAWAY and close the connection. # CLI flag: -server.grpc.keepalive.ping-without-stream-allowed [grpc_server_ping_without_stream_allowed: \u003cboolean\u003e | default = true] # Output log messages in the given format. Valid formats: [logfmt, json] # CLI flag: -log.format [log_format: \u003cstring\u003e | default = \"logfmt\"] # Only log messages with the given severity or above. Valid levels: [debug, # info, warn, error] # CLI flag: -log.level [log_level: \u003cstring\u003e | default = \"info\"] # Optionally log the source IPs. # CLI flag: -server.log-source-ips-enabled [log_source_ips_enabled: \u003cboolean\u003e | default = false] # Header field storing the source IPs. Only used if # server.log-source-ips-enabled is true. If not set the default Forwarded, # X-Real-IP and X-Forwarded-For headers are used # CLI flag: -server.log-source-ips-header [log_source_ips_header: \u003cstring\u003e | default = \"\"] # Regex for matching the source IPs. Only used if server.log-source-ips-enabled # is true. If not set the default Forwarded, X-Real-IP and X-Forwarded-For # headers are used # CLI flag: -server.log-source-ips-regex [log_source_ips_regex: \u003cstring\u003e | default = \"\"] # Optionally log request headers. # CLI flag: -server.log-request-headers [log_request_headers: \u003cboolean\u003e | default = false] # Optionally log requests at info level instead of debug level. Applies to # request headers as well if server.log-request-headers is enabled. # CLI flag: -server.log-request-at-info-level-enabled [log_request_at_info_level_enabled: \u003cboolean\u003e | default = false] # Comma separated list of headers to exclude from loggin. Only used if # server.log-request-headers is true. # CLI flag: -server.log-request-headers-exclude-list [log_request_exclude_headers_list: \u003cstring\u003e | default = \"\"] # Base path to serve all API routes from (e.g. /v1/) # CLI flag: -server.path-prefix [http_path_prefix: \u003cstring\u003e | default = \"\"] storage_config The storage_config configures the storage type Cortex uses.\n# The storage engine to use: blocks is the only supported option today. # CLI flag: -store.engine [engine: \u003cstring\u003e | default = \"blocks\"] store_gateway_config The store_gateway_config configures the store-gateway service used by the blocks storage.\n# Shard blocks across multiple store gateway instances. This option needs be set # both on the store-gateway and querier when running in microservices mode. # CLI flag: -store-gateway.sharding-enabled [sharding_enabled: \u003cboolean\u003e | default = false] # The hash ring configuration. This option is required only if blocks sharding # is enabled. sharding_ring: # The key-value store used to share the hash ring across multiple instances. # This option needs be set both on the store-gateway and querier when running # in microservices mode. kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -store-gateway.sharding-ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -store-gateway.sharding-ring.prefix [prefix: \u003cstring\u003e | default = \"collectors/\"] dynamodb: # Region to access dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -store-gateway.sharding-ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: store-gateway.sharding-ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: store-gateway.sharding-ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -store-gateway.sharding-ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -store-gateway.sharding-ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -store-gateway.sharding-ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -store-gateway.sharding-ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -store-gateway.sharding-ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 15s] # The heartbeat timeout after which store gateways are considered unhealthy # within the ring. 0 = never (timeout disabled). This option needs be set both # on the store-gateway and querier when running in microservices mode. # CLI flag: -store-gateway.sharding-ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # The replication factor to use when sharding blocks. This option needs be set # both on the store-gateway and querier when running in microservices mode. # CLI flag: -store-gateway.sharding-ring.replication-factor [replication_factor: \u003cint\u003e | default = 3] # File path where tokens are stored. If empty, tokens are not stored at # shutdown and restored at startup. # CLI flag: -store-gateway.sharding-ring.tokens-file-path [tokens_file_path: \u003cstring\u003e | default = \"\"] # True to enable zone-awareness and replicate blocks across different # availability zones. # CLI flag: -store-gateway.sharding-ring.zone-awareness-enabled [zone_awareness_enabled: \u003cboolean\u003e | default = false] # True to keep the store gateway instance in the ring when it shuts down. The # instance will then be auto-forgotten from the ring after # 10*heartbeat_timeout. # CLI flag: -store-gateway.sharding-ring.keep-instance-in-the-ring-on-shutdown [keep_instance_in_the_ring_on_shutdown: \u003cboolean\u003e | default = false] # Minimum time to wait for ring stability at startup. 0 to disable. # CLI flag: -store-gateway.sharding-ring.wait-stability-min-duration [wait_stability_min_duration: \u003cduration\u003e | default = 1m] # Maximum time to wait for ring stability at startup. If the store-gateway # ring keeps changing after this period of time, the store-gateway will start # anyway. # CLI flag: -store-gateway.sharding-ring.wait-stability-max-duration [wait_stability_max_duration: \u003cduration\u003e | default = 5m] # Timeout for waiting on store-gateway to become desired state in the ring. # CLI flag: -store-gateway.sharding-ring.wait-instance-state-timeout [wait_instance_state_timeout: \u003cduration\u003e | default = 10m] # The sleep seconds when store-gateway is shutting down. Need to be close to # or larger than KV Store information propagation delay # CLI flag: -store-gateway.sharding-ring.final-sleep [final_sleep: \u003cduration\u003e | default = 0s] # Name of network interface to read address from. # CLI flag: -store-gateway.sharding-ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # The availability zone where this instance is running. Required if # zone-awareness is enabled. # CLI flag: -store-gateway.sharding-ring.instance-availability-zone [instance_availability_zone: \u003cstring\u003e | default = \"\"] # The sharding strategy to use. Supported values are: default, shuffle-sharding. # CLI flag: -store-gateway.sharding-strategy [sharding_strategy: \u003cstring\u003e | default = \"default\"] # Comma separated list of tenants whose store metrics this storegateway can # process. If specified, only these tenants will be handled by storegateway, # otherwise this storegateway will be enabled for all the tenants in the # store-gateway cluster. # CLI flag: -store-gateway.enabled-tenants [enabled_tenants: \u003cstring\u003e | default = \"\"] # Comma separated list of tenants whose store metrics this storegateway cannot # process. If specified, a storegateway that would normally pick the specified # tenant(s) for processing will ignore them instead. # CLI flag: -store-gateway.disabled-tenants [disabled_tenants: \u003cstring\u003e | default = \"\"] hedged_request: # If true, hedged requests are applied to object store calls. It can help with # reducing tail latency. # CLI flag: -store-gateway.hedged-request.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of hedged requests allowed for each initial request. A high # number can reduce latency but increase internal calls. # CLI flag: -store-gateway.hedged-request.max-requests [max_requests: \u003cint\u003e | default = 3] # It is used to calculate a latency threshold to trigger hedged requests. For # example, additional requests are triggered when the initial request response # time exceeds the 90th percentile. # CLI flag: -store-gateway.hedged-request.quantile [quantile: \u003cfloat\u003e | default = 0.9] tracing_config The tracing_config configures backends cortex uses.\n# Tracing type. OTEL and JAEGER are currently supported. For jaeger # `JAEGER_AGENT_HOST` environment variable should also be set. See: # https://cortexmetrics.io/docs/guides/tracing . # CLI flag: -tracing.type [type: \u003cstring\u003e | default = \"jaeger\"] otel: # otl collector endpoint that the driver will use to send spans. # CLI flag: -tracing.otel.otlp-endpoint [otlp_endpoint: \u003cstring\u003e | default = \"\"] # enhance/modify traces/propagators for specific exporter. If empty, OTEL # defaults will apply. Supported values are: `awsxray.` # CLI flag: -tracing.otel.exporter-type [exporter_type: \u003cstring\u003e | default = \"\"] # Fraction of traces to be sampled. Fractions \u003e= 1 means sampling if off and # everything is traced. # CLI flag: -tracing.otel.sample-ratio [sample_ratio: \u003cfloat\u003e | default = 0.001] # If enabled, use round_robin gRPC load balancing policy. By default, use # pick_first policy. For more details, please refer to # https://github.com/grpc/grpc/blob/master/doc/load-balancing.md#load-balancing-policies. # CLI flag: -tracing.otel.round-robin [round_robin: \u003cboolean\u003e | default = false] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -tracing.otel.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] tls: # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -tracing.otel.tls.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -tracing.otel.tls.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. # If not set, the host's root CA certificates are used. # CLI flag: -tracing.otel.tls.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -tracing.otel.tls.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -tracing.otel.tls.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] LimitsPerLabelSet limits: # The maximum number of active series per LabelSet, across the cluster before # replication. Setting the value 0 will enable the monitoring (metrics) but # would not enforce any limits. [max_series: \u003cint\u003e | default = ] # LabelSet which the limit should be applied. [label_set: \u003cmap of string (labelName) to string (labelValue)\u003e | default = []] PriorityDef # Priority level. Must be a unique value. [priority: \u003cint\u003e | default = 0] # Number of reserved queriers to handle priorities higher or equal to the # priority level. Value between 0 and 1 will be used as a percentage. [reserved_queriers: \u003cfloat\u003e | default = 0] # List of query_attributes to match and assign priority to queries. A query is # assigned to this priority if it matches any query_attribute in this list. Each # query_attribute has several properties (e.g., regex, time_window, user_agent), # and all specified properties must match for a query_attribute to be considered # a match. Only the specified properties are checked, and an AND operator is # applied to them. [query_attributes: \u003clist of QueryAttribute\u003e | default = []] QueryAttribute # API type for the query. Should be one of the query, query_range, series, # labels, label_values. If not set, it won't be checked. [api_type: \u003cstring\u003e | default = \"\"] # Regex that the query string (or at least one of the matchers in metadata # query) should match. If not set, it won't be checked. [regex: \u003cstring\u003e | default = \"\"] # Overall data select time window (including range selectors, modifiers and # lookback delta) that the query should be within. If not set, it won't be # checked. time_window: # Start of the data select time window (including range selectors, modifiers # and lookback delta) that the query should be within. If set to 0, it won't # be checked. [start: \u003cint\u003e | default = 0] # End of the data select time window (including range selectors, modifiers and # lookback delta) that the query should be within. If set to 0, it won't be # checked. [end: \u003cint\u003e | default = 0] # Query time range should be within this limit to match. Depending on where it # was used, in most of the use-cases, either min or max value will be used. If # not set, it won't be checked. time_range_limit: # This will be duration (12h, 1d, 15d etc.). Query time range should be above # or equal to this value to match. Ex: if this value is 20d, then queries # whose range is bigger than or equal to 20d will match. If set to 0, it won't # be checked. [min: \u003cint\u003e | default = 0] # This will be duration (12h, 1d, 15d etc.). Query time range should be below # or equal to this value to match. Ex: if this value is 24h, then queries # whose range is smaller than or equal to 24h will match.If set to 0, it won't # be checked. [max: \u003cint\u003e | default = 0] # If query step provided should be within this limit to match. If not set, it # won't be checked. This property only applied to range queries and ignored for # other types of queries. query_step_limit: # Query step should be above or equal to this value to match. If set to 0, it # won't be checked. [min: \u003cint\u003e | default = 0] # Query step should be below or equal to this value to match. If set to 0, it # won't be checked. [max: \u003cint\u003e | default = 0] # Regex that User-Agent header of the request should match. If not set, it won't # be checked. [user_agent_regex: \u003cstring\u003e | default = \"\"] # Grafana includes X-Dashboard-Uid header in query requests. If this field is # provided then X-Dashboard-Uid header of request should match this value. If # not set, it won't be checked. This property won't be applied to metadata # queries. [dashboard_uid: \u003cstring\u003e | default = \"\"] # Grafana includes X-Panel-Id header in query requests. If this field is # provided then X-Panel-Id header of request should match this value. If not # set, it won't be checked. This property won't be applied to metadata queries. [panel_id: \u003cstring\u003e | default = \"\"] DisabledRuleGroup # namespace in which the rule group belongs [namespace: \u003cstring\u003e | default = \"\"] # name of the rule group [name: \u003cstring\u003e | default = \"\"] ","categories":"","description":"","excerpt":" Cortex can be configured using a YAML file - specified using the …","ref":"/docs/configuration/configuration-file/","tags":"","title":"Configuration file"},{"body":" Author: Christian Simon Date: October 2020 Status: Accepted Overview This document aims to describe how to implement the ability to allow queries to cover data from more than a single Cortex tenant.\nReasoning Adopting a tenancy model within an organization with each tenant representing a department comes with the disadvantage that it will prevent queries from spanning multiple departments. This proposal tries to overcome those limitations.\nAlternatives considered Aggregation in PromQL API clients In theory PromQL API clients could be aggregating/correlating query results from multiple tenants. For example Grafana could be used with multiple data sources and a cross tenant query could be achieved through using Transformations.\nAs this approach comes with the following disadvantages, it was not considered further:\nEvery PromQL API client needs to support the aggregation from various sources.\nQueries that are written in PromQL can’t be used without extra work across tenants.\nMulti tenant aggregation in the query frontends Another approach to multi tenant query federation could be achieved by aggregation of partial query results within the query frontend. For this a query needs to be split into sub queries per tenant and afterwards the partial results need reduced into the final result.\nThe astmapper package goes down a similar approach, but it cannot parallelize all query types. Ideally multi-tenant query federation should support the full PromQL language and the algorithms necessary would differ per query functions and operators used. This approach was deemed as a fairly complex way to achieve that tenant query federation.\nChallenges Aggregate data without overlaps Challenge The series in different tenants might have exactly the same labels and hence potentially collide which each other.\nProposal In order to be able to always identify the tenant correctly, queries using multiple tenants should inject a tenant label named __tenant_id__ and its value containing the tenant ID into the results. A potentially existing label with the same name should be stored in a label with the prefix original_.\nLabel selectors containing the tenant label should behave like any other labels. This can be achieved by selecting the tenants used in a multi tenant query.\nExposure of feature to the user Challenge The tenant ID is currently read from the X-Scope-OrgID HTTP header. The tenant ID has those documented limitations of values being allowed.\nProposal For the query path a user should be able to specify a X-Scope-OrgID header with multiple tenant IDs. Multiple tenant IDs should then be propagating throughout out the system until it reaches the querier. The Queryable instance returned by the querier component, is wrapped by a mergeQueryable, which will aggregate the results from a Queryable per tenant and hence treated by the downstream components as a single tenant query.\nTo allow such queries to be processed we suggest that an experimental configuration flag -querier.tenant-federation.enabled will be added, which is switched off by default. Once enabled the value of the X-Scope-OrgID header should be interpreted as | separated list of tenant ids. Components which are not expecting multiple tenant ids (e.g. the ingress path) must signal an error if multiple are used.\nImplementing Limits, Fairness and Observability for Cross-Tenant queries Challenge In Cortex the tenant id is used as the primary identifier for those components:\nThe limits that apply to a certain query.\nThe query-frontend maintains a per tenant query queue to implement fairness.\nRelevant metrics about the query are exposed under a user label.\nHaving a query spanning multiple tenants, the existing methods are no longer correct.\nProposal The identifier for aforementioned features for queries involving more than a single tenant should be derived from: An ordered, distinct list of tenant IDs, which is joined by a |. This will produce a reproducible identifier for the same set of tenants no matter which order they have been specified.\nWhile this feature is considered experimental, this provides some insights and ability to limit multi-tenant queries with these short comings:\nCardinality costs to the possible amount of tenant ID combinations.\nQuery limits applied to single tenants part of a multi tenant query are ignored.\nConclusion Challenge Status Aggregate data without overlap Implementation in PR #3250 Exposure of feature to the user Implementation in PR #3250 Implementing Limits, Fairness and Observability for Cross-Tenant queries Implementation in PR #3250 Future work Those features are currently out of scope for this proposal, but we can foresee some interest implementing them after this proposal.\nCross Tenant support for the ruler Ability to use multi tenant queries in the ruler.\nAllow the identifier for limits, fairness and observability to be switched out It would be great if the source identifier could be made more pluggable. This could allow to for example base all of those features on another dimension (e.g. users rather than tenants)\nAllow customisation of the label used to expose tenant ids As per this proposal the label name __tenant_id__ is fixed, but users might want to be able to modify that through a configuration option.\nRetain overlapping tenant id label values recursively As per this proposal the tenant label injection retains an existing label value, but this is not implemented recursively. So if the result already contains __tenant_id__ and original__tenant_id__ labels, the value of the latter would be lost.\n","categories":"","description":"","excerpt":" Author: Christian Simon Date: October 2020 Status: Accepted Overview …","ref":"/docs/proposals/cross-tenant-query-federation/","tags":"","title":"Cross-Tenant Query Federation"},{"body":" Author: Peter Stibrany Date: November 2020 Status: Accepted Deletion of tenant data Problem When a tenant is deleted from the external system that controls access to Cortex, we want to clean up tenants data from Cortex as well.\nBackground When using blocks storage, Cortex stores tenant’s data in several places:\nobject store for long-term storage of blocks, ingesters disks for short-term storage. Ingesters eventually upload data to long-term storage, various caches: query-frontend, chunks, index and metadata, object store for rules (separate from blocks), object store for alert manager configuration, state for alertmanager instances (notifications and silences). This document expects that there is an external authorization system in place. Disabling or deleting the tenant in this system will stop tenants data, queries and other API calls from reaching Cortex. Note that there may be a delay before disabling or deleting the user, until data / queries fully stop, due to eg. caching in authorization proxies. Cortex endpoint for deleting a tenant data should be only called after this blocking is in place.\nProposal API Endpoints POST /purger/delete_tenant We propose to introduce an /purger/delete_tenant API endpoint to trigger data deletion for the tenant.\nWhile this endpoint works as an “admin” endpoint and should not be exposed directly to tenants, it still needs to know under which tenant it should operate. For consistency with other endpoints this endpoint would therefore require an X-Scope-OrgID header and use the tenant from it.\nIt is safe to call “delete_tenant” multiple times.\nGET /purger/delete_tenant_status To monitor the state of data deletion, another endpoint will be available: /purger/delete_tenant_status. This will return OK if tenant’s data has been fully deleted – no more blocks exist on the long-term storage, alertmanager and rulers are fully stopped and configuration removed. Similarly to “delete_tenant” endpoint, “delete_tenant_status” will require X-Scope-OrgID header.\nImplementation (asynchronous) Purger will implement both API endpoints. Upon receiving the call to /purger/delete_tenant endpoint, Purger will initiate the deletion by writing “deletion marker” objects for specific tenant to following buckets:\nBlocks bucket Ruler configuration bucket Alertmanager configuration bucket Deletion marker for the tenant will be an object stored under tenant prefix, eg. “//deleted”. This object will contain the timestamp when it was created, so that we can delete it later based on the specified time period. We could reuse a subset of the proposed Thanos tombstones format, or use custom format.\n“delete_tenant_status” endpoint will report success, if all of the following are true:\nThere are no blocks for the tenant in the blocks bucket There is a “deletion finished” object in the bucket for the tenant in the ruler configuration bucket. There is a “deletion finished” object in the bucket for the tenant in the alertmanager configuration bucket. See later sections on Ruler and Alertmanager for explanation of “deletion finished” objects.\nBlocks deletion marker Blocks deletion marker will be used by compactor, querier and store-gateway.\nCompactor Upon discovering the blocks deletion marker, the compactor will start deletion of all blocks that belong to the tenant. This can take hours to finish, depending on the number of blocks. Even after deleting all blocks on the storage, ingesters may upload additional blocks for the tenant. To make sure that the compactor deletes these blocks, the compactor will keep the deletion marker in the bucket. After a configurable period of time it can delete the deletion marker too.\nTo implement deletion, Compactor should use new TenantsCleaner component similar to existing BlocksCleaner (which deletes blocks marked for deletion), and modify UserScanner to ignore deleted tenants (for BlocksCleaner and Compactor itself) or only return deleted tenants (for TenantsCleaner).\nQuerier, Store-gateway These two components scan for all tenants periodically, and can use the tenant deletion marker to skip tenants and avoid loading their blocks into memory and caching to disk (store-gateways). By implementing this, store-gateways will also unload and remove cached blocks.\nQueriers and store-gateways use metadata cache and chunks cache when accessing blocks to reduce the number of API calls to object storage. In this proposal we don’t suggest removing obsolete entries from the cache – we will instead rely on configured expiration time for cache items.\nNote: assuming no queries will be sent to the system once the tenant is deleted, implementing support for tenant deletion marker in queriers and store-gateways is just an optimization. These components will unload blocks once they are deleted from the object store even without this optimization.\nQuery Frontend While query-frontend could use the tenant blocks deletion marker to clean up the cache, we don’t suggest to do that due to additional complexity. Instead we will only rely on eventual eviction of cached query results from the cache. It is possible to configure Cortex to set TTL for items in the frontend cache by using -frontend.default-validity option.\nIngester Ingesters don’t scan object store for tenants.\nTo clean up the local state on ingesters, we will implement closing and deletion of local per-tenant data for idle TSDB. (See Cortex PR #3491). This requires additional configuration for ingesters, specifically how long to wait before closing and deleting TSDB. This feature needs to work properly in two different scenarios:\nIngester is no longer receiving data due to ring changes (eg. scale up of ingesters) Data is received because user has been deleted. Ingester doesn’t distinguish between the two at the moment. To make sure we don’t break queries by accidentally deleting TSDB too early, ingester needs to wait at least -querier.query-ingesters-within duration.\nAlternatively, ingester could check whether deletion marker exists on the block storage, when it detects idle TSDB.\nIf more data is pushed to the ingester for a given tenant, ingester will open new TSDB, build new blocks and upload them. It is therefore essential that no more data is pushed to Cortex for the tenant after calling the “delete_tenant” endpoint.\nRuler deletion marker This deletion marker is stored in the ruler configuration bucket. When rulers discover this marker during the periodic sync of the rule groups, they will\nstop the evaluation of the rule groups for the user, delete tenant rule groups (when multiple rulers do this at the same time, they will race for deletion, and need to be prepared to handle possible “object not found” errors) delete local state. When the ruler is finished with this cleanup, it will ask all other rulers if they still have any data for the tenant. If all other rulers reply with “no”, ruler can write “deletion finished” marker back to the bucket. This allows rulers to ignore the ruler completely, and it also communicates the status of the deletion back to purger.\nNote that ruler currently relies on cached local files when using Prometheus Ruler Manager component. This can be avoided now, and since it makes cleanup simpler, we suggest to modify Cortex ruler implementation to avoid this local copy.\nSimilarly to ingesters, it is necessary to disable access to the ruler API for deleted tenant. This must be done in external authorization proxy.\nAlertmanager deletion marker Deletion marker for alert manager is stored in the alertmanager configuration bucket. Cleanup procedure for alertmanager data is similar to rulers – when individual alertmanager instances discover the marker, they will:\nDelete tenant configuration Delete local notifications and silences state Ask other alertmanagers if they have any tenant state yet, and if not, write “deletion finished” marker back to the bucket. To perform the last step, Alertmanagers need to find other alertmanager instances. This will be implemented by using the ring, which will (likely) be added as per Alertmanager scaling proposal.\nAccess to Alertmanager API must be disabled for tenant that is going to be deleted.\nAlternatives Considered Another possibility how to deal with tenant data deletion is to make purger component actively communicate with ingester, compactor, ruler and alertmanagers to make data deletion faster. In this case purger would need to understand how to reach out to all those components (with multiple ring configurations, one for each component type), and internal API calls would need to have strict semantics around when the data deletion is complete. This alternative has been rejected due to additional complexity and only small benefit in terms of how fast data would be deleted.\n","categories":"","description":"","excerpt":" Author: Peter Stibrany Date: November 2020 Status: Accepted Deletion …","ref":"/docs/proposals/tenant-deletion/","tags":"","title":"Deletion of Tenant Data from Blocks Storage"},{"body":" Cortex Cortex is a horizontally scalable, highly available, multi-tenant, long term storage for Prometheus.\nFeatures Horizontally scalable: Cortex can run across multiple machines in a cluster, exceeding the throughput and storage of a single machine. Highly available: When run in a cluster, Cortex can replicate data between machines. Multi-tenant: Cortex can isolate data and queries from multiple different independent Prometheus sources in a single cluster. Long term storage: Cortex supports S3, GCS, Swift and Microsoft Azure for long term storage of metric data. Getting Started To get started with Cortex, follow these steps:\nInstallation Basic Setup Example Usage Documentation Architecture Overview Configuration Guide Security Guide Contributor Guidelines Community and Support If you have any questions about Cortex, you can:\nAsk a question on the Cortex Slack channel. To invite yourself to the CNCF Slack, visit http://slack.cncf.io/. File an issue. Email cortex-users@lists.cncf.io. Your feedback is always welcome.\nFor security issues see https://github.com/cortexproject/cortex/security/policy\nEngage with Our Community We invite you to participate in the Cortex Community Calls, an exciting opportunity to connect with fellow developers and enthusiasts. These meetings are held every 4 weeks on Thursdays at 1700 UTC, providing a platform for open discussion, collaboration, and knowledge sharing.\nOur meeting notes are meticulously documented and can be accessed here, offering a comprehensive overview of the topics discussed and decisions made.\nTo ensure you never miss a meeting, we’ve made it easy for you to keep track:\nView the Cortex Community Call schedule in your browser here. Alternatively, download the .ics file here for use with any calendar application or service that supports the iCal format. Join us in shaping the future of Cortex, and let’s build something amazing together!\nFurther reading Talks Nov 2024 KubeCon talk “Cortex Intro: Multi-Tenant Scalable Prometheus” (video, slides) Mar 2024 KubeCon talk “Cortex Intro: Multi-Tenant Scalable Prometheus” (video, slides) Apr 2023 KubeCon talk “How to Run a Rock Solid Multi-Tenant Prometheus” (video, slides) Oct 2022 KubeCon talk “Current State and the Future of Cortex” (video, slides) Oct 2021 KubeCon talk “Cortex: Intro and Production Tips” (video, slides) Sep 2020 KubeCon talk “Scaling Prometheus: How We Got Some Thanos Into Cortex” (video, slides) Jul 2020 PromCon talk “Sharing is Caring: Leveraging Open Source to Improve Cortex \u0026 Thanos” (video, slides) Nov 2019 KubeCon talks “Cortex 101: Horizontally Scalable Long Term Storage for Prometheus” (video, slides), “Configuring Cortex for Max Performance” (video, slides, write up) and “Blazin’ Fast PromQL” (slides, video, write up) Nov 2019 PromCon talk “Two Households, Both Alike in Dignity: Cortex and Thanos” (video, slides, write up) May 2019 KubeCon talks; “Cortex: Intro” (video, slides, blog post) and “Cortex: Deep Dive” (video, slides) Nov 2018 CloudNative London meetup talk; “Cortex: Horizontally Scalable, Highly Available Prometheus” (slides) Aug 2018 PromCon panel; “Prometheus Long-Term Storage Approaches” (video) Dec 2018 KubeCon talk; “Cortex: Infinitely Scalable Prometheus” (video, slides) Aug 2017 PromCon talk; “Cortex: Prometheus as a Service, One Year On” (videos, slides, write up part 1, part 2, part 3) Jun 2017 Prometheus London meetup talk; “Cortex: open-source, horizontally-scalable, distributed Prometheus” (video) Dec 2016 KubeCon talk; “Weave Cortex: Multi-tenant, horizontally scalable Prometheus as a Service” (video, slides) Aug 2016 PromCon talk; “Project Frankenstein: Multitenant, Scale-Out Prometheus”: (video, slides) Blog Posts Dec 2020 blog post “How AWS and Grafana Labs are scaling Cortex for the cloud” Oct 2020 blog post “How to switch Cortex from chunks to blocks storage (and why you won’t look back)” Oct 2020 blog post “Now GA: Cortex blocks storage for running Prometheus at scale with reduced operational complexity” Sep 2020 blog post “A Tale of Tail Latencies” Aug 2020 blog post “Scaling Prometheus: How we’re pushing Cortex blocks storage to its limit and beyond” Jul 2020 blog post “How blocks storage in Cortex reduces operational complexity for running Prometheus at massive scale” Mar 2020 blog post “Cortex: Zone Aware Replication” Mar 2020 blog post “How we’re using gossip to improve Cortex and Loki availability” Jan 2020 blog post “The Future of Cortex: Into the Next Decade” Feb 2019 blog post \u0026 podcast; “Prometheus Scalability with Bryan Boreham” (podcast) Feb 2019 blog post; “How Aspen Mesh Runs Cortex in Production” Dec 2018 CNCF blog post; “Cortex: a multi-tenant, horizontally scalable Prometheus-as-a-Service” Nov 2018 CNCF TOC Presentation; “Horizontally Scalable, Multi-tenant Prometheus” (slides) Sept 2018 blog post; “What is Cortex?” Jul 2018 design doc; “Cortex Query Optimisations” Jun 2016 design document; “Project Frankenstein: A Multi Tenant, Scale Out Prometheus” Hosted Cortex There are several commercial services where you can use Cortex on-demand:\nAmazon Managed Service for Prometheus (AMP) Amazon Managed Service for Prometheus (AMP) is a Prometheus-compatible monitoring service that makes it easy to monitor containerized applications at scale. It is a highly available, secure, and managed monitoring for your containers. Get started here. To learn more about the AMP, reference our documentation and Getting Started with AMP blog.\nEmeritus Maintainers Peter Štibraný @pstibrany Marco Pracucci @pracucci Bryan Boreham @bboreham Goutham Veeramachaneni @gouthamve Jacob Lisi @jtlisi Tom Wilkie @tomwilkie Alvin Lin @alvinlin123 History of Cortex The Cortex project was started by Tom Wilkie and Julius Volz (Prometheus’ co-founder) in June 2016.\n","categories":"","description":"","excerpt":" Cortex Cortex is a horizontally scalable, highly available, …","ref":"/docs/","tags":"","title":"Documentation"},{"body":" Author: Jay Batra Date: March 2020 Status: proposal Problem In Cortex, currently, we are missing versioning of documentation. The idea is to have version documentation just like Prometheus.Prometheus. Documentation is the main source of information for current contributors and first-timers. A properly versioned documentation will help everyone to have a proper place to look for answers before flagging it in the community.\nIn this proposal, we want to solve this. In particular, we want to:\nVersion specific pages of the documentation Include links to change version (the version must be in the URL) Include the master version and last 3 minor releases. Documentation defaults to the last minor release. Proposed solution Currently, the documentation is residing under the docs/ folder of cortexproject/cortex. It is built by Hugo using the theme docsy. It will have a proper drop-down menu which will enable proper versioning. It has a section params.version in config.toml which will allow us to map URLs with proper versions. We will have to change all the occurrences of older doc links with new links. We will keep master version with 3 latest release versions. Each release is a minor version expressed as 1.x. The document would default to latest minor version.\nFrom the current doc, the following paths (and all their subpages) should be versioned for now:\nhttps://cortexmetrics.io/docs/apis/ https://cortexmetrics.io/docs/configuration/ (moving v1.x Guarantees outside of the tree, because these shouldn’t be versioned) The above should be versioned under a single URL path (/docs/running-cortex/ in the following example, but final prefix is still to be decided).\nExample: For master version we would be able to use the above links via the following path\n/docs/running-cortex/master/configuration/ /docs/running-cortex/master/api/ And for a minor version like 1.x:\n/docs/running-cortex/1.0/configuration/ /docs/running-cortex/1.0/apis/ we’ll have versioned documentation only under the /docs/running-cortex/ prefix and, as a starting point, all versioned pages should go there.\n","categories":"","description":"","excerpt":" Author: Jay Batra Date: March 2020 Status: proposal Problem In …","ref":"/docs/proposals/documentation-versioning/","tags":"","title":"Documentation Versioning"},{"body":" Author: @annanay25 Reviewers: @jtlisi, @pstibrany, @cyriltovena, @pracucci Date: April 2020 Status: Accepted Overview Cortex uses modules to start and operate services with dependencies. Inter-service dependencies are specified in a map and passed to a module manager which ensures that they are initialised in the right order of dependencies. While this works really well, the implementation is tied in specifically to the Cortex struct and is not flexible for use with other projects like Loki, which also require similar forms of dependency management.\nWe would like to extend modules in cortex to a generic dependency management framework, that can be used by any project with no ties to cortex.\nSpecific goals Framework should allow for reusing cortex modules and allow us to: Add new modules Overwrite the implementation of a current module Manage dependencies Framework should allow for building an application from scratch using the modules package, with no dependencies on Cortex. For ex: Remove code from Loki that was copied from pkg/cortex/cortex.go. Proposed Design Modules package To make the modules package extensible, we need to abstract away any Cortex specific details from the module manager. The proposed design is to:\nMake a new component Manager, which is envisioned to be a central manager for all modules of the application. It stores modules \u0026 dependencies, and will be housed under a new package pkg/util/modules. Manager has the following methods for interaction: func (m *Manager) RegisterModule(name string, initFn func() (Service, error)) func (m *Manager) AddDependency(name string, dependsOn... string) error func (m *Manager) InitModuleServices(target string) (map[string]services.Service, error) Modules can be created by the application and registered with modules.Manager using RegisterModule. The parameters are:\nname: Name of the module initFn: A function that will be used to start the module. If it returns nil, and other modules depend on it, InitModuleServices will return an error. Dependencies between modules can be added using AddDependency. The parameters to the function are:\nname: Name of the module dependsOn: A variadic list of modules that the module depends on. These need to be added before the call to InitModuleServices.\nThe application can be initialized by running initFn’s of all the modules in the right order of dependencies by invoking InitModuleServices with the target module name.\nChanges to pkg/cortex: WrappedService present in the current module design will be deprecated. All initFn’s will be wrapped into WrappedService by default.\nWhile the process of loading modules into modules.Manager should be remain as part of the Cortex.New() function, InitModuleServices should be part of Cortex.Run() and to enable this, modules.Manager would be made a member of the Cortex struct.\nUsage Following these changes, the Modules package will be a generic dependency management framework that can be used by any project.\nTo use the modules framework: Import the pkg/util/modules package, and initialize a new instance of the Manager using modules.NewManager() Create components in the system that implement the services interface (present in pkg/util/services). Register each of these components as a module using Manager.RegisterModule() by passing name of the module and initFn for the module. To add dependencies between modules, use Manager.AddDependency() Once all modules are added into modules.Manager, initialize the application by calling Manager.InitModuleServices() which initializes modules in the right order of dependencies. Future work Extend the module manager to allow specifying multiple targets as opposed to a single target name supported currently. Factor out Run() method to make it independent of Cortex. This will help reduce replicated code in the Loki project as well as help manage modules.Manager outside of the Cortex struct. ","categories":"","description":"","excerpt":" Author: @annanay25 Reviewers: @jtlisi, @pstibrany, @cyriltovena, …","ref":"/docs/proposals/generalize-modules/","tags":"","title":"Generalize Modules Service to make it extensible"},{"body":"Cortex is a powerful platform software that can be run in two modes: as a single binary or as multiple independent microservices.\nThere are two guides in this section:\nSingle Binary Mode with Docker Compose Microservice Mode with KIND The single binary mode is useful for testing and development, while the microservice mode is useful for production.\nBoth guides will help you get started with Cortex using blocks storage.\nSingle Binary Mode This guide will help you get started with Cortex in single-binary mode using blocks storage.\nPrerequisites Cortex can be configured to use local storage or cloud storage (S3, GCS, and Azure). It can also utilize external Memcached and Redis instances for caching. This guide will focus on running Cortex as a single process with no dependencies.\nDocker Compose Running Cortex as a Single Instance For simplicity, we’ll start by running Cortex as a single process with no dependencies. This mode is not recommended or intended for production environments or production use.\nThis example uses Docker Compose to set up:\nAn instance of SeaweedFS for S3-compatible object storage An instance of Cortex to receive metrics. An instance of Prometheus to send metrics to Cortex. An instance of Grafana to visualize the metrics. Instructions $ git clone https://github.com/cortexproject/cortex.git $ cd cortex/docs/getting-started Start the services # Start SeaweedFS to emulate S3 storage for Cortex. $ docker-compose up seaweedfs -d --wait # Create buckets in SeaweedFS. $ for bucket in cortex-blocks cortex-ruler cortex-alertmanager; do curl --aws-sigv4 \"aws:amz:local:seaweedfs\" --user \"any:any\" -X PUT http://localhost:8333/$bucket done # Start the remaining services. $ docker-compose up -d --wait We can now access the following services:\nCortex Prometheus Grafana SeaweedFS If everything is working correctly, Prometheus should be sending metrics that it is scraping to Cortex. Prometheus is configured to send metrics to Cortex via remote_write. Check out the prometheus-config.yaml file to see how this is configured.\nConfigure Cortex Recording Rules and Alerting Rules We can configure Cortex with cortextool to load recording rules and alerting rules. This is optional, but it is helpful to see how Cortex can be configured to manage rules and alerts.\n# Configure recording rules for the Cortex tenant (optional) $ docker run --network host -v \"$(pwd):/workspace\" -w /workspace quay.io/cortexproject/cortex-tools:v0.17.0 rules sync rules.yaml alerts.yaml --id cortex --address http://localhost:9009 Configure Cortex Alertmanager Cortex also comes with a multi-tenant Alertmanager. Let’s load configuration for it to be able to view them in Grafana.\n# Configure alertmanager for the Cortex tenant $ docker run --network host -v \"$(pwd):/workspace\" -w /workspace quay.io/cortexproject/cortex-tools:v0.17.0 alertmanager load alertmanager-config.yaml --id cortex --address http://localhost:9009 You can configure Alertmanager in Grafana as well.\nThere’s a list of recording rules and alerts that should be visible in Grafana here.\nExplore Grafana is configured to use Cortex as a data source. Grafana is also configured with Cortex Dashboards to understand the state of the Cortex instance. The dashboards are generated from the cortex-jsonnet repository. There is a Makefile in the repository that can be used to update the dashboards.\n# Update the dashboards (optional) $ make If everything is working correctly, then the metrics seen in Grafana were successfully sent from Prometheus to Cortex via remote_write!\nOther things to explore:\nCortex - Administrative interface for Cortex Try shutting down the ingester, and see how it affects metric ingestion. Restart Cortex to bring the ingester back online, and see how Prometheus catches up. Does it affect the querying of metrics in Grafana? Prometheus - Prometheus instance that is sending metrics to Cortex Try querying the metrics in Prometheus. Are they the same as what you see in Cortex? Grafana - Grafana instance that is visualizing the metrics. Try creating a new dashboard and adding a new panel with a query to Cortex. Clean up $ docker-compose down Microservice Mode Now that you have Cortex running as a single instance, let’s explore how to run Cortex in microservice mode.\nPrerequisites Kind kubectl Helm This example uses Kind to set up:\nA Kubernetes cluster An instance of SeaweedFS for S3-compatible object storage An instance of Cortex to receive metrics An instance of Prometheus to send metrics to Cortex An instance of Grafana to visualize the metrics Setup Kind $ kind create cluster Configure Helm $ helm repo add cortex-helm https://cortexproject.github.io/cortex-helm-chart $ helm repo add grafana https://grafana.github.io/helm-charts $ helm repo add prometheus-community https://prometheus-community.github.io/helm-charts Instructions $ git clone https://github.com/cortexproject/cortex.git $ cd cortex/docs/getting-started Configure SeaweedFS (S3) # Create a namespace $ kubectl create namespace cortex # We can emulate S3 with SeaweedFS $ kubectl --namespace cortex apply -f seaweedfs.yaml --wait --timeout=5m # Wait for SeaweedFS to be ready $ sleep 5 $ kubectl --namespace cortex wait --for=condition=ready pod -l app=seaweedfs --timeout=5m # Port-forward to SeaweedFS to create a bucket $ kubectl --namespace cortex port-forward svc/seaweedfs 8333 \u0026 # Create buckets in SeaweedFS $ for bucket in cortex-blocks cortex-ruler cortex-alertmanager; do curl --aws-sigv4 \"aws:amz:local:seaweedfs\" --user \"any:any\" -X PUT http://localhost:8333/$bucket done Setup Cortex # Deploy Cortex using the provided values file which configures # - blocks storage to use the seaweedfs service $ helm upgrade --install --version=2.4.0 --namespace cortex cortex cortex-helm/cortex -f cortex-values.yaml --wait Setup Prometheus # Deploy Prometheus to scrape metrics in the cluster and send them, via remote_write, to Cortex. $ helm upgrade --install --version=25.20.1 --namespace cortex prometheus prometheus-community/prometheus -f prometheus-values.yaml --wait If everything is working correctly, Prometheus should be sending metrics that it is scraping to Cortex. Prometheus is configured to send metrics to Cortex via remote_write. Check out the prometheus-config.yaml file to see how this is configured.\nSetup Grafana # Deploy Grafana to visualize the metrics that were sent to Cortex. $ helm upgrade --install --version=7.3.9 --namespace cortex grafana grafana/grafana -f grafana-values.yaml --wait # Create dashboards for Cortex $ for dashboard in $(ls dashboards); do basename=$(basename -s .json $dashboard) cmname=grafana-dashboard-$basename kubectl create --namespace cortex configmap $cmname --from-file=$dashboard=dashboards/$dashboard --save-config=true -o yaml --dry-run=client | kubectl apply -f - kubectl patch --namespace cortex configmap $cmname -p '{\"metadata\":{\"labels\":{\"grafana_dashboard\":\"\"}}}' done # Port-forward to Grafana to visualize $ kubectl --namespace cortex port-forward deploy/grafana 3000 \u0026 View the dashboards in Grafana.\nConfigure Cortex Recording Rules and Alerting Rules (Optional) We can configure Cortex with cortextool to load recording rules and alerting rules. This is optional, but it is helpful to see how Cortex can be configured to manage rules and alerts.\n# Port forward to the alertmanager to configure recording rules and alerts $ kubectl --namespace cortex port-forward svc/cortex-nginx 8080:80 \u0026 # Configure recording rules for the cortex tenant $ cortextool rules sync rules.yaml alerts.yaml --id cortex --address http://localhost:8080 Configure Cortex Alertmanager (Optional) Cortex also comes with a multi-tenant Alertmanager. Let’s load configuration for it to be able to view them in Grafana.\n# Configure alertmanager for the cortex tenant $ cortextool alertmanager load alertmanager-config.yaml --id cortex --address http://localhost:8080 You can configure Alertmanager in Grafana as well.\nThere’s a list of recording rules and alerts that should be visible in Grafana here.\nExplore Grafana is configured to use Cortex as a data source. Grafana is also configured with Cortex Dashboards to understand the state of the Cortex instance. The dashboards are generated from the cortex-jsonnet repository. There is a Makefile in the repository that can be used to update the dashboards.\n# Update the dashboards (optional) $ make If everything is working correctly, then the metrics seen in Grafana were successfully sent from Prometheus to Cortex via remote_write!\nOther things to explore:\nCortex - Administrative interface for Cortex\n# Port forward to the ingester to see the administrative interface for Cortex $ kubectl --namespace cortex port-forward deploy/cortex-ingester 9009:8080 \u0026 Try shutting down the ingester, and see how it affects metric ingestion. Restart Cortex to bring the ingester back online, and see how Prometheus catches up. Does it affect the querying of metrics in Grafana? Prometheus - Prometheus instance that is sending metrics to Cortex\n# Port forward to Prometheus to see the metrics that are being scraped $ kubectl --namespace cortex port-forward deploy/prometheus-server 9090 \u0026 Try querying the metrics in Prometheus. Are they the same as what you see in Cortex? Grafana - Grafana instance that is visualizing the metrics.\n# Port forward to Grafana to visualize $ kubectl --namespace cortex port-forward deploy/grafana 3000 \u0026 Try creating a new dashboard and adding a new panel with a query to Cortex. Clean up # Remove the port-forwards $ killall kubectl $ kind delete cluster ","categories":"","description":"","excerpt":"Cortex is a powerful platform software that can be run in two modes: …","ref":"/docs/getting-started/","tags":"","title":"Getting Started"},{"body":"Gojek launched in 2010 as a call center for booking motorcycle taxi rides in Indonesia. Today, the startup is a decacorn serving millions of users across Southeast Asia with its mobile wallet, GoPay, and 20+ products on its super app. Want to order dinner? Book a massage? Buy movie tickets? You can do all of that with the Gojek app.\nThe company’s mission is to solve everyday challenges with technology innovation. To achieve that across multiple markets the systems team at Gojek focused on building an infrastructure for speed, reliability, and scale. By 2019, the team realized it needed a new monitoring system that could keep up with Gojek’s ever-growing technology organization, which led them to Cortex, the horizontally scalable Prometheus implementation.\n“We were using InfluxDB for metrics storage. Developers configured alerts by committing kapacitor scripts in git repos. To achieve high availability, we had a relay setup with two InfluxDBs. Since we could not horizontally scale Influx unless we paid for an enterprise license, we ended up having many InfluxDB clusters with relay setup,” says Product Engineer Ankit Goel.\nThough the team had introduced automation for setup, managing all those Influx instances became a pain point for operations. Additionally, some of the Gojek engineering teams needed far greater scale. “Some of our teams generate more than a million active time series,” says Goel. Another common requirement from customers was long-term storage of metrics. With InfluxDB, Gojek only had 2 weeks’ retention, and increasing it would mean provisioning bigger instances.\nGojek was in search of a better monitoring solution that would meet the following requirements:\nKubernetes native. Horizontally scalable. Highly available out of the box. High reliability. Low operations overhead so a small team can manage it. Cortex met all of these requirements, and also had the following features that the Gojek team could leverage:\nMulti-tenancy. Customizable and modifiable, so it could be integrated with Gojek’s existing tooling. Support for remote_write. Because it supports remote_write, Cortex enabled one of Gojek’s key needs: the ability to offer monitoring as a service. “With Thanos, we would have had to deploy a Thanos sidecar on every Prometheus that would have been deployed,” says Goel. “So essentially, there would be a substantial part of infrastructure on the client side that we would need to manage. We preferred Cortex because people could simply push their metrics to us, and we would have all the metrics in a single place.”\nThe implementation started in January 2019. The team developed a few tools: a simple service for token-based authentication, and another for storing team information, such as notification channels and PagerDuty policies. Once all this was done, they leveraged InfluxData Telegraf’s remote_write plugin to write to Cortex. This allowed them to have all the metrics being sent to InfluxDB to be sent to Cortex as well. “So moving from InfluxDB to tenants would not be that complicated. Because Cortex was multi-tenant, we could directly map each of our InfluxDB servers to our tenants,” says Goel. They’ve developed an internal helm chart to deploy and manage Cortex. After the customizations were completed in about two months, “we had our setup up and running, and we onboarded one team on Cortex,” he says.\nIn the initial version, GitOps was the only interface for developers to apply alerts and create dashboards. The team built tools like grafonnet-playground to make it easy for developers to create dashboards. Developers are also allowed to create dashboards using the UI, since Grafana maintains version history for dashboards.\n“We needed metrics like ‘the number of alerts triggered for each team,’ ‘how long did it take to resolve these alerts,’ ‘how many were actionable and how many were ignored,’ etc.,” says Goel. “For measuring these metrics, the team only had to create a simple dashboard, since the ruler component exposes the per-tenant alert metrics. Both business and developers have found these metrics to be very useful.”\nThe team built a CLI tool to improve user experience for applying alerts without having to dig into PromQL. “You can write a command and say lens attach alert, and you tell it what kind of alert you want to attach, such as a CPU alert or Postgres alerts, and then you give it a service name,” says Goel. “There are some challenges to this approach for applying alerts, but we would like to move to such a model in the future.”\nOne of the challenges the team faces is developer education. But “we always knew if we are going to move to either Thanos or Cortex, developers would have to learn PromQL,” Goel says. The monitoring team paired with developers to help them understand PromQL and migrate their graphs and alerts.\nThe monitoring team has faced issues with Cortex from time to time, but “we always reached out to the Cortex community with our issues through the Cortex slack channel,” says Goel, and “active members of the Cortex community have always helped us with our problems.”\nToday, Gojek’s Lens monitoring system has 40+ tenants, for which Cortex handles about 1.2 million samples per second. Adoption is growing organically by word of mouth. Gojek is currently migrating to Kubernetes, and the teams that moved to Kubernetes have found Prometheus to be a better fit than InfluxDB. Seeing that success, other teams on Kubernetes have onboarded themselves to Lens.\nUltimately, Goel says, “where Cortex has really helped us is to integrate the monitoring system with our existing tools. We have a lot of internal tooling, and in certain places, we needed really tight integrations with the monitoring system. So the goal is to make sure that whenever a new service or team is created, they automatically get onboarded to the monitoring platform. After developers deploy, some of their system metrics and all the other standard metrics that are available for a service are automatically sent to the platform.” The team plans to spend the next six months bringing everyone over to Lens.\nLooking ahead, Goel and his team have the long-term vision of growing from a monitoring team to a full-fledged observability team. “We also want to take care of logging and tracing in Gojek,” he says. “Loki would be easy to fit with Cortex, so in the future we want to explore Loki for logging.”\n","categories":"","description":"","excerpt":"Gojek launched in 2010 as a call center for booking motorcycle taxi …","ref":"/docs/case-studies/gojek/","tags":"","title":"How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale"},{"body":"This document defines project governance for the project.\nVoting The Cortex project employs voting to ensure no single member can dominate the project. Any maintainer may cast a vote. To avoid having a single company dominate the project, at most two votes from maintainers working for the same company will count.\nFor formal votes, a specific statement of what is being voted on should be added to the relevant github issue or PR, and a link to that issue or PR added to the maintainers meeting agenda document.\nMaintainers should indicate their yes/no vote on that issue or PR, and after a suitable period of time (minimum 2 business weeks), the votes will be tallied and the outcome noted. Maintainers who do not cast a vote, after a suitable period of time, are not included in the majority calculation.\nMaintainer Role Maintainers are required to participate in the project, by joining discussions, submitting and reviewing pull requests, answering user questions, among others.\nBesides that, we have one concrete activity in which maintainers have to engage from time to time: releasing new versions of Cortex. This process ideally takes only a couple of hours, but requires coordination on different fronts. Even though the process is well documented, it is not without eventual glitches, so, each release needs a “Release shepherd”. How it works is described in the RELEASE.md file.\nChanges in Maintainership Contributors who are interested in becoming a maintainer, if performing relevant responsibilities, should discuss their interest with the existing maintainers. New maintainers must be nominated by an existing maintainer and must be elected by 2/3 majority vote.\nWe do not expect anyone to make a permanent commitment to be a Cortex maintainer forever. After all, circumstances change, people get new jobs, new interests, and may not be able to continue contributing to the project. At the same time, we need to keep the list of maintainers current in order to have effective governance. People may be removed from the current list of maintainers via one of the following ways:\nThey can resign If they stop contributing to the project for a period of 6 months or more By a 2/3 majority vote from active maintainers Former maintainers are recognized with an honorary Emeritus Maintainer status, and have their names permanently listed in the README as a form of gratitude for their contributions.\nApproving PRs PRs may be merged after receiving at least two positive votes. If the PR author is a maintainer, this counts as a vote.\nTriage Role Triagers are contributors who assist in managing and organizing issues and pull requests. See here for details about the role’s permission.\nGithub Project Administration Maintainers will be added to the collaborators list of the Cortex repository with “Write” access while Triagers will be added to the collaborators list of the Cortex repository with “Triage” access.\nAfter 6 months a maintainer will be given “Admin” access to the Cortex repository.\nChanges in Governance All changes in Governance require a 2/3 majority vote.\nCortex Team Maintainers Name Email GitHub Company Alan Protasio alanprot@gmail.com @alanprot Amazon Web Services Ben Ye yb532204897@gmail.com @yeya24 Amazon Web Services Charlie Le chtle4@gmail.com\u003e @CharlieTLe Apple Daniel Blando daniel@blando.com.br @danielblando Amazon Web Services Friedrich Gonzalez friedrichg@gmail.com @friedrichg Adobe Triaggers Name Email GitHub Company Sungjin Lee tjdwls1201@gmail.com @SungJin1212 KakaoEnterprise Other Changes Unless specified above, all other changes to the project require a 2/3 majority vote. Additionally, any maintainer may request that any change require a 2/3 majority vote.\n","categories":"","description":"","excerpt":"This document defines project governance for the project.\nVoting The …","ref":"/docs/contributing/governance/","tags":"","title":"Governance"},{"body":" Author: @jtlisi Reviewers: @pracucci, @pstibrany, @khaines, @gouthamve Date: March 2020 Status: Accepted Overview The purpose of this design document is to propose a set of standards that should be the basis of the Cortex HTTP API. This document will outline the current state of the Cortex http api and describe limitations that result from the current approach. It will also outline a set of paradigms on how http routes should be created within Cortex.\nCurrent Design As things currently stand, the majority of HTTP API calls exist under the /api/prom path prefix. This prefix is configurable. However, since this prefix is shared between all the modules which leads to conflicts if the Alertmanager is attempted to be run as as part of the single binary (#1722).\nProposed Design Module-Based Routing Cortex incorporates three separate APIs: Alertmanager, Prometheus, and Cortex. Each of these APIs should use a separate route prefix that accurately describes the API. Currently, all of the api calls in Cortex reside under the configured http prefix. Instead the following routing tree is proposed:\n/prometheus/* Under this path prefix, Cortex will act as a Prometheus web server. It will host all of the required Prometheus api endpoints. For example to query cortex the endpoint /prometheus/api/v1/query_range will be used.\n/alertmanager/* Under this path prefix, Cortex will act as a Alertmanager web server. In this case, it will forward requests to the alertmanager and support the alertmanager API. This means for a user to access their Alertmanager UI, they will use the /alertmanager path of cortex.\n/api/v1/* – The cortex API will exist under this path prefix. /push /chunks /rules/* Current Proposed /api/prom/push /api/v1/push /api/prom/chunks /api/v1/chunks /api/prom/rules/* /api/v1/rules/* Service Endpoints A number of endpoints currently exist that are not under the /api/prom prefix that provide basic web interfaces and trigger operations for cortex services. These endpoints will all be placed under a url with their service name as a prefix if it is applicable.\nCurrent Proposed /status /multitenant-alertmanager/status /config /config /ring /ingester/ring /ruler_ring /ruler/ring /compactor/ring /compactor/ring /store-gateway/ring /store-gateway/ring /ha-tracker /distributor/ha_tracker /all_user_stats /distributor/all_user_stats /user_stats /distributor/user_stats /flush /ingester/flush /shutdown /ingester/shutdown Path Versioning Cortex will utilize path based versioning similar to both Prometheus and Alertmanager. This will allow future versions of the API to be released with changes over time.\nBackwards-Compatibility The new API endpoints and the current http prefix endpoints can be maintained concurrently. The flag to configure these endpoints will be maintained as http.prefix. This will allow us to roll out the new API without disrupting the current routing schema. The original http prefix endpoints can maintained indefinitely or be phased out over time. Deprecation warnings can be added to the current API either when initialized or utilized. This can be accomplished by injecting a middleware that logs a warning whenever a legacy API endpoint is used.\nIn cases where Cortex is run as a single binary, the Alertmanager module will only be accessible using the new API.\nImplementation This will be implemented by adding an API module to the Cortex service. This module will handle setting up all the required HTTP routes with Cortex. It will be designed around a set of interfaces required to fulfill the API. This is similar to how the v1 Prometheus API is implemented.\nStyle All new paths will utilize _ instead of - for their url to conform with Prometheus and its use of the underscore in the query_range endpoint. This applies to all operations endpoints. Component names in the path can still contain dashes. For example: /store-gateway/ring. ","categories":"","description":"","excerpt":" Author: @jtlisi Reviewers: @pracucci, @pstibrany, @khaines, …","ref":"/docs/proposals/http-api-design/","tags":"","title":"HTTP API Design"},{"body":" Author: @pstibrany Reviewers: Date: June 2020 Status: Replaced with migration guide (now removed from this site). Warning Suggestions from this proposal were implemented, but general procedure outlined here doesn’t quite work in Kubernetes environment. Please see chunks to blocks migration guide (now removed from this site) instead.\nIntroduction This short document describes the first step in full migration of the Cortex cluster from using chunks storage to using blocks storage, specifically switching ingesters to using blocks, and modification of queriers to query both chunks and blocks storage.\nIngesters When switching ingesters from chunks to blocks, we need to consider the following:\nIngesting of new data, and querying should work during the switch. Ingesters are rolled out with new configuration over time. There is overlap: ingesters of both kinds (chunks, blocks) are running at the same time. Ingesters using WAL don’t flush in-memory chunks to storage on shutdown. Rollout should be as automated as possible. How do we handle ingesters with WAL (non-WAL ingesters are discussed below)? There are several possibilities, but the simplest option seems to be adding a new flag to ingesters to flush chunks on shutdown. This is trivial change to ingester, and allows us to do automated migration by:\nEnabling this flag on each ingester (first rollout). Turn off chunks, enable TSDB (second rollout). During the second rollout, as the ingester shuts down, it will flush all chunks in memory, and when it restarts, it will start using TSDB. Benefit of this approach is that it is trivial to add the flag, and then rollout in both steps can be fully automated. In this scenario, we will reconfigure existing statefulset of ingesters to use blocks in step 2.\nNotice that querier can ask only ingesters for most recent data and not consult the store, but during the rollout (and some time after), ingesters that are already using blocks will not have the most recent chunks in memory. To make sure queries work correctly, -querier.query-store-after needs to be set to 0, in order for queriers to not rely on ingesters only for most recent data. After couple of hours after rollout, this value can be increased again, depending on how much data ingesters keep. (-experimental.blocks-storage.tsdb.retention-period for blocks, -ingester.retain-period for chunks) During the rollout, chunks and blocks ingesters share the ring and use the same statefulset.\nOther alternatives considered for flushing chunks / handling WAL:\nReplay chunks-WAL into TSDB head on restart. In this scenario, chunks-ingester shuts down, and block ingester starts. It can detect existing chunks WAL, and replay it into TSDB head (and then delete old WAL). Issue here is that current chunks-WAL is quite specific to ingester code, and would require some refactoring to make this happen. Deployment is trivial: just reconfigure ingesters to start using blocks, and replay chunks WAL if found. Required change seems like a couple of days of coding work, but it is essentially only used once (for each cluster). Doesn’t seem like good time investment. Shutdown single chunks-ingester, run flusher in its place, and when done start new blocks ingester. This is similar to the procedure we did during the introduction of WAL. Flusher can be run via initContainer support in pods. This still requires two-step deployment: 1) enable flusher and reconfigure ingesters to use blocks, 2) remove flusher. When not using WAL, ingesters using chunks cannot transfer those chunks to new ingesters that start with blocks support, so old ingesters need to be configured to disable transfers (using -ingester.max-transfer-retries=0), and to flush chunks on shutdown instead. As ingesters without WAL are typically deployed using Kubernetes deployment, while blocks ingesters need to use statefulset, and there is no chunks transfer happening, it is possible to configure and start blocks-ingesters and then stop old deployment.\nAfter all ingesters are converted to blocks, we can set cut-off time for querying chunks storage on queriers.\nFor rollback from blocks to chunks, we need to be able to flush data from ingesters to the blocks storage, and then switch ingesters back to chunks. Ingesters are currently not able to flush blocks to storage, but adding flush-on-shutdown option, support for /shutdown endpoint and support in flusher component similar to chunks is doable, and should be part of this work.\nWith this ability, rollback would follow the same process, just in reverse: 1) redeploy with flush flag enabled, 2a) redeploy with config change from blocks to chunks (when using WAL) or 2b) scale down statefulset with blocks-ingesters, and start deployment with chunk-ingesters again. Note that this isn’t a full rollback to chunks-only solution, as generated blocks still need to be queried after the rollback, otherwise samples pushed to blocks would be missing. This means running store-gateways and queriers that can query both chunks and blocks store.\nAlternative plan could be to use a separate Cortex cluster configured to use blocks, and redirect incoming traffic to both chunks and blocks cluster. When one is confident about the blocks cluster running correctly, old chunks cluster can be shutdown. In this plan, there is an overlap where both clusters are ingesting same data. Blocks cluster needs to be configured to be able to query chunks storage as well, with cut-off time based on when clusters were configured (at latest, to minimize amount of duplicated samples that need to be processed during queries.)\nQuerying To be able to query both old and new data, querier needs to be modified to be able to query both blocks (on object store only) and chunks store (NoSQL + object store) at the same time, and merge results from both.\nFor querying chunks storage, we have two options:\nAlways query the chunks store – useful during ingesters switch, or after rollback from blocks to chunks. Query chunk store only for queries that ask for data after specific cut-off time. This is useful after all ingesters have switched, and we know the timestamp since ingesters are only writing blocks. Querier needs to support both modes of querying chunks store. Which one of these two modes is used depends on single timestamp flag passed to the querier. If timestamp is configured, chunks store is only used for queries that ask for data older than timestamp. If timestamp is not configured, chunks store is always queried.\nFor blocks, we don’t need to use the timestamp flag. Queriers can always query blocks – each querier knows about existing blocks and their timeranges, so it can quickly determine whether there are any blocks with relevant data. Always querying blocks is also useful when there is some background process converting chunks to blocks. As new blocks with old data appear on the store as a result of conversion, they get queried if necessary.\nWhile we could use runtime-config for on-the-fly switch without restarts, queriers restart quickly and so switching via configuration or command line option seems enough.\nWork to do Ingester: Add flags for always flushing on shutdown, even when using WAL or blocks. Querier: Add support for querying both chunk store and blocks at the same time and test the support for querying both chunks and blocks from ingesters works correctly Querier: Add cut-off time support to querier to query chunk the store only if needed, based on query time. ","categories":"","description":"","excerpt":" Author: @pstibrany Reviewers: Date: June 2020 Status: Replaced with …","ref":"/docs/proposals/ingesters-migration/","tags":"","title":"Migrating ingesters from chunks to blocks and back."},{"body":" Author: Roy Chiang Date: May 2021 Status: Proposed Introduction As a part of pushing Cortex’s scaling capability at AWS, we have done performance testing with Cortex and found the compactor to be one of the main limiting factors for higher active timeseries limit per tenant. The documentation Compactor describes the responsibilities of a compactor, and this proposal focuses on the limitations of the current compactor architecture. In the current architecture, compactor has simple sharding, meaning that a single tenant is sharded to a single compactor. The compactor generates compaction groups, which are groups of Prometheus TSDB blocks that can be compacted together, independently of another group. However, a compactor currently handles compaction groups of a single tenant iteratively, meaning that blocks belonging non-overlapping times are not compacted in parallel.\nCortex ingesters are responsible for uploading TSDB blocks with data emitted by a tenant. These blocks are considered as level-1 blocks, as they contain duplicate timeseries for the same time interval, depending on the replication factor. Vertical compaction is done to merge all the blocks with the same time interval and deduplicate the samples. These merged blocks are level-2 blocks. Subsequent compactions such as horizontal compaction can happen, further increasing the compaction level of the blocks.\nProblem and Requirements Currently, a compactor is able to compact up to 20M timeseries within 2 hours for a level-2 compaction, including the time to download blocks, compact, and upload the newly compacted block. We would like to increase the timeseries limit per tenant, and compaction is one of the limiting factors. In addition, we would like to achieve the following:\nCompact multiple non-overlapping time intervals concurrently, so we can achieve higher throughput for the compaction of a single tenant We should be able to scale up, down compactor as needed, depending on how many compactions are pending Insight into the compaction progress of a tenant, such as the number of compactions required in order to catch up to the newest blocks Design We accept the fact that a single compaction can potentially take more than 2 hours to compact, and we achieve higher compaction throughput through horizontally scaling the compactor. To compact more blocks in parallel for a single tenant, we distribute the compaction groups to compactors, instead of introducing more parallelism within a compactor.\nParallelize Work This proposal builds heavily on top of the GrafanaLabs approach of introducing parallelism via time intervals. The difference being that a single tenant is now sharded across multiple compactors instead of just a single compactor. The initial approach will be to work on distinct time intervals, but the compactor planner can be later extended to introduce parallelism within a time interval as well.\nThe following is an example of parallelize work at each level:\nCompactors are shuffle-sharded, meaning that 1 tenant can belong to multiple compactors, and these subset of compactors determine which blocks should be compacted together. Compactors determine amongst themselves the responsibility of the compaction blocks, by using a hash of time interval and tenant id, and putting it on the sharding ring.\nThe benefit of this approach is that this aligns with what Cortex currently does in Ruler. The downside is that a compaction job can only be assigned to a single compactor, rather than all of the compactors sharded for the tenant. If a compaction job takes forever, other tenants sharded to the same compactor will be blocked until the issue is resolved. With the scheduler approach, any compactor assigned to a given tenant can pick up any work required.\nScenarios Bad block resulting in non-ideal compaction groups A Cortex operator configures the compaction block range as 2h and 6h. If a full 6-hour block cannot be compacted due to compaction failures, the compactor should not split up the group into subgroups, as this may cause suboptimal grouping of block. Cortex has full information regarding all the available blocks, so we should utilize this information to achieve the best compaction group possible.\nAlternatives Shard compaction jobs amongst compactors with a scheduler We add a new component Compactor Scheduler, which is responsible for calculating the compaction plan, and distributing compaction groups to compactors. The planner is sharded by tenant id, so that we can horizontally scale the planner as needed in order to accept more tenants in the cortex cluster. A tenant will have two queues inside the planner, a compaction queue and a clean up queue, similar to how the query frontend currently holds queues of pending queries.\nOnce a compactor scheduler pushes a job to a compactor, the job is no longer available. Every set interval, or once the compaction is done, a compactor will update the compactor schedule the current status of the compaction job. If a compactor does not provide an update to the scheduler within a timeout, the compaction job becomes available to be assigned to other compactors.\nConcurrency To achieve concurrency within a single tenant, compactor scheduler will push jobs to compactors. Compactors are shuffle-sharded by tenant id, to prevent a large tenant from impacting the compaction of other tenants. Compactor will download blocks from long term storage, compact, and upload. Compactor will also pull from the clean up queues from scheduler, and delete blocks marked for deletion.\nConsistency On resharding of compactor schedulers, a tenant might move to a different scheduler. We can either drop the current compactor job in order to prevent duplicate compaction jobs, or continue compaction. I propose that the compactor drops the compaction job if the compaction group no longer belongs to the original compactor scheduler. This way, we do not have duplicate compactions happening, and we can minimize work wasted.\nContribute to Thanos for a more scalable compactor Instead of introducing parallelism on the Cortex compactor level, we move the parallelism to the Thanos compactor itself. Thanos has a proposal to make compactor more scalable, and a PR. Cortex will enjoy higher throughput per tenant if Thanos is able to speed up the compaction, and we can keep the Cortex architecture the same. However, this approach means that a single tenant is still sharded to a single compactor. In order to compact more groups at once, we must scale up compactor vertically. Although vertical scaling can get us far, we should scale horizontally where we can.\n","categories":"","description":"","excerpt":" Author: Roy Chiang Date: May 2021 Status: Proposed Introduction As a …","ref":"/docs/proposals/parallel-compaction/","tags":"","title":"Parallel Compaction by Time Interval"},{"body":" Author: Allenzhli Date: January 2021 Status: Accepted, Implemented in PR #3879. Retention of tenant data Problem Metric data is growing over time per-tenant, at the same time, the value of data decreases. We want to have a retention policy like prometheus does. In Cortex, data retention is typically achieved via a bucket policy. However, this has two main issues:\nNot every backend storage support bucket policies Bucket policies don’t easily allow a per-tenant custom retention Background tenants When using blocks storage, Cortex stores tenant’s data in object store for long-term storage of blocks, tenant id as part of the object store path. We discover all tenants via scan the root dir of bucket.\nruntime config Using the “overrides” mechanism (part of runtime config) already allows for per-tenant settings. See runtime-configuration-file for more details. Using it for tenant retention would fit nicely. Admin could set per-tenant retention here, and also have a single global value for tenants that don’t have custom value set.\nProposal retention period field We propose to introduce just one new field RetentionPeriod in the Limits struct(defined at pkg/util/validation/limits.go).\nRetentionPeriod setting how long historical metric data retention period per-tenant. 0 is disable.\nRuntime config is reloaded periodically (defaults to 10 seconds), so we can update the retention settings on-the-fly.\nFor each tenant, if a tenant-specific runtime_config value exists, it will be used directly, otherwise, if a default limits_config value exists, then the default value will be used; If neither exists, do nothing.\nImplementation A BlocksCleaner within the Compactor run periodically (which defaults to 15 minutes) and the retention logic will insert into it. The logic should compare retention value to block maxTime and blocks that match maxTime \u003c now - retention will be marked for delete.\nBlocks deletion is not immediate, but follows a two steps process. See soft-and-hard-blocks-deletion\n","categories":"","description":"","excerpt":" Author: Allenzhli Date: January 2021 Status: Accepted, Implemented in …","ref":"/docs/proposals/tenant-retention/","tags":"","title":"Retention of Tenant Data from Blocks Storage"},{"body":" Author: Daniel Blando Date: August 2022 Status: Proposed Background Cortex implements a ring structure to share information of registered pods for each service. The data stored and used by the ring need to be implemented via Codec interface. Currently, the only supported Codec to the ring is Desc. Desc is a proto.Message with a list of instances descriptions. It is used to store the data for each pod and saved on a supported KV store. Currently, Cortex supports memberlist, consul and etcd as KV stores. Memberlist works implementing a gossip protocol while consul and etcd are a KV store service.\nThe ring is used by different services using a different ring key to store and receive the values from the KV store. For example, ingester service uses the key “ingester” to save and load data from KV. As the saved data is a Desc struct only one key is used for all the information.\nProblem Each service using a single key to save and load information creates a concurrency issue when multiple pods are saving the same key. When using memberlist, the issue is mitigate as the information is owned by all pods and timestamp is used to confirm latest data. For consul and etcd, all pods compete to update the key at the same time causing an increase on latency and failures direct related to number of pods running. Cortex and etcd implementation use a version tag to make sure no data is being overridden causing the problem of write failures.\nOn a test running cortex with etcd, distributor was scaled to 300 pods and latency increase was noticed coming from etcd usage. We can also notice 5xx happening when etcd was running. 17:14 - Running memberlist, p99 around 5ms 17:25 - Running etcd, p99 around 200ms 17:25 to 17:34 migrating to multikey After running etcd multikey poc, p99 around 25ms\nProposal Multikey interface The proposal is separate the current Desc struct which contains a list of key value in multiple keys. Instead of saving one “ingester” key, the KV store will have “ingester-1”, “ingester-2” keys saved.\nCurrent:\nKey: ingester/ring/ingester Value: { \"ingesters\": { \"ingester-0\": { \"addr\": \"10.0.0.1:9095\", \"timestamp\": 1660760278, \"tokens\": [ 1, 2 ], \"zone\": \"us-west-2b\", \"registered_timestamp\": 1660708390 }, \"ingester-1\": ... } } Proposal:\nKey: ingester/ring/ingester-0 Value: { \"addr\": \"10.0.0.1:9095\", \"timestamp\": 1660760278, \"tokens\": [ 1, 15 ], \"zone\": \"us-west-2b\", \"registered_timestamp\": 1660708390 } Key: ingester/ring/ingester-1 Value: { \"addr\": \"10.0.0.2:9095\", \"timestamp\": 1660760378, \"tokens\": [ 5, 28 ], \"zone\": \"us-west-2b\", \"registered_timestamp\": 1660708572 } The proposal is to create an interface called MultiKey. The interface allows KV store to request the codec to split and join the values is separated keys.\ntype MultiKey interface { SplitById() map[string]interface{} JoinIds(map[string]interface{}) Multikey GetChildFactory() proto.Message FindDifference(MultiKey) (Multikey, []string, error) } SplitById - responsible to split the codec in multiple keys and interface. JoinIds - responsible to receive multiple keys and interface creating the codec object GetChildFactory - Allow the kv store to know how to serialize and deserialize the interface returned by “SplitById”. The interface returned by SplitById need to be a proto.Message FindDifference - optimization used to know what need to be updated or deleted from a codec. This avoids updating all keys every time the coded change. First parameter returns a subset of the Multikey to be updated. Second is a list of keys to be deleted. The codec implementation will change to support multiple keys. Currently, the codec interface for KV store supports only Encode and Decode. New methods will be added which would be used only by the KV stores implementing the multi key functionality.\ntype Codec interface { //Existen Decode([]byte) (interface{}, error) Encode(interface{}) ([]byte, error) CodecID() string //Proposed DecodeMultiKey(map[string][]byte) (interface{}, error) EncodeMultiKey(interface{}) (map[string][]byte, error) } DecodeMultiKey - called by KV store to decode data downloaded. This function will use the JoinIds method. EncodeMultiKey - called by KV store to encode data to be saved. This function will use the SplitById method. The new KV store will know the data being saved is a reference for multikey. It will use the FindDifference to know which keys need to be updated. The codec implementation for the new methods will use the JoinIds and SplitById to know how to separate the codec in multiple keys. The DecodeMultiKey will also use GetChildFactory to know how to decode the data stored in the kv store.\nExample of CAS being used with multikey design: ","categories":"","description":"","excerpt":" Author: Daniel Blando Date: August 2022 Status: Proposed Background …","ref":"/docs/proposals/ring-multikey/","tags":"","title":"Ring Multikey"},{"body":" Author: Soon-Ping Phang Date: June 2022 Status: Deprecated Introduction This proposal is deprecated in favor of the new proposal\nThis proposal consolidates multiple existing PRs from the AWS team working on this feature, as well as future work needed to complete support. The hope is that a more holistic view will make for more productive discussion and review of the individual changes, as well as provide better tracking of overall progress.\nThe original issue is #4435.\nProblem Rulers in Cortex currently run with a replication factor of 1, wherein each RuleGroup is assigned to exactly 1 ruler. This lack of redundancy creates the following risks:\nRule group evaluation Missed evaluations due to a ruler outage, possibly caused by a deployment, noisy neighbour, hardware failure, etc. Missed evaluations due to a ruler brownout due to other tenant rule groups sharing the same ruler (noisy neighbour) API -inconsistent API results during resharding (e.g. due to a deployment) when rulers are in a transition state loading rule groups This proposal attempts to mitigate the above risks by enabling a ruler replication factor of greater than 1, allowing multiple rulers to evaluate the same rule group — effectively, the ruler equivalent of ingester HA already supported in Cortex.\nProposal Make ReplicationFactor configurable ReplicationFactor in Ruler is currently hardcoded to 1. Making this a configurable parameter is the first step to enabling HA in ruler, and would also be the mechanism for the user to turn the feature on. The parameter value will be 1 by default, equating to the feature being turned off by default.\nA replication factor greater than 1 will result in the same rules being evaluated by multiple ruler instances.\nThis redundancy will allow for missed or skipped rule evaluations from single ruler outages to be covered by other instances evaluating the same rules.\nThere is also the question of duplicate metrics generated by replicated evaluation. We do not expect this to be a problem, as all replicas must use the same slotted intervals when evaluating rules, which should result in the same timestamp applying to metrics generated by each replica, with the samples being deduplicated in the ingestion path.\nPR #4712 [open]\nAdjustments for ALERTS_FOR_STATE When an alert fires in a rulegroup shared across multiple rulers, the alert on each ruler will have slightly different timestamps due to the differences in evaluation time on each ruler. This results in the ALERTS_FOR_STATE metric being deduplicated with the “duplicate sample for timestamp” error. To prevent this, we synchronize Alert.activeAt for the same alerts in different rulers using a post-processing hook.\nPRs:\nPrometheus PR #9665 [merged] Prometheus PR #10070 [merged] Cortex PR #4712 [open] Weak and Strong Quorum in ListRules and ListAlerts ListRules/ListAlerts will return inconsistent responses while a new configuration propagates to multiple ruler HA instances. For most users, this is an acceptable side-effect of an eventually consistent HA architecture. However, some use-cases have stronger consistency requirements and are willing to sacrifice some availability for those use-cases. For example, an alerts client might want to verify that a change has propagated to a majority of instances before informing the user that the update succeeded. To enable this, we propose adding an optional quorum API parameter with the following behaviour:\nquorum=weak (default)\nBiased towards availability, ListRules will perform a best-effort merge of the results from at least $i = tenantShardSize - replicationFactor + 1$ instances. If a rulegroup definition does not satisfy a quorum of $q = \\lfloor{replicationFactor / 2}\\rfloor + 1$ copies, it will choose the most recently evaluated version of that rulegroup for the final result set. An error response will be returned only if all instances return an error. Note that a side-effect of this rule is that the API will return a result even if all but one ruler instances in the tenant’s shard is unavailable. quorum=strong\nBiased towards consistency, ListRules will query at least $i = tenantShardSize - \\lfloor{replicationFactor / 2}\\rfloor + 1$ instances. If any rulegroup does not satisfy a quorum of $q = \\lfloor{replicationFactor / 2}\\rfloor + 1$ copies, a 503 error will be returned. PR #4768 [open]\nAlternatives to a quorum API parameter Weak or strong quorum by default Another option for making ListRules work in HA mode is to implement one of the quorum rules (weak or strong) as the default, with no control to select between the two, outside of maybe a configuration parameter. AWS itself runs multitenant Cortex instances, and we have an internal use-case for the strong quorum implementation, but we do not want to impose the subsequent availability hit on our customers, particularly given that replication_factor is not currently a tenant-specific parameter in Cortex, for ingesters, alert manager, or ruler.\nMaking HA availability the default, while giving users the choice to knowingly request for more consistency at the cost of more error-handling seems like a good balance.\n","categories":"","description":"","excerpt":" Author: Soon-Ping Phang Date: June 2022 Status: Deprecated …","ref":"/docs/proposals/ruler-ha/","tags":"","title":"Ruler HA"},{"body":" Author: Anand Rajagopal Date: Aug 2024 Status: Proposed Problem Rulers in Cortex currently run with a replication factor of 1, wherein each RuleGroup is assigned to exactly 1 ruler. This lack of redundancy creates the following risks:\nRule group evaluation Missed evaluations due to a ruler outage, possibly caused by a deployment, noisy neighbour, hardware failure, etc. Missed evaluations due to a ruler brownout due to other tenant rule groups sharing the same ruler (noisy neighbour) API Inconsistent API results during resharding (e.g. due to a deployment) when rulers are in a transition state loading rule groups This proposal attempts to mitigate the above risks by enabling a ruler replication factor of greater than 1, allowing multiple rulers to evaluate the same rule group — effectively.\nProposal Make ReplicationFactor configurable ReplicationFactor in Ruler is currently hardcoded to 1. Making this a configurable parameter is the first step to enabling HA in ruler. The parameter value will be 1 by default. To enable Ruler HA for rule group evaluation, a new flag will be created\nA replication factor greater than 1 will result in the following\nRing will pick R rulers for a rule group where R=RF The primary ruler (R1), when active, will take ownership of the rule group Non-primary ruler R2 will check if R1 is active. If R1 is not active, R2 will take ownership of the rule group Non-primary ruler R3 (if RF=3) will check if R1 and R2 are active. If they are both inactive/unhealthy, then R3 will take owership of the rule group Non-primary rulers will drop their ownership when R1 becomes active after an outage With this redundancy, the maximum duration of missed evaluations will be limited to the sync interval of the rule groups, reducing the impact of primary Ruler unavailability.\nPrometheus change No Prometheus change is required for this proposal\nAPI HA An interim solution is addressed in this #5773 PR. This will be modified such that the replicas will return both active and passive rule groups and the API handler will continue to de-duplicate the results. The difference is that after Ruler HA, the replicas could potentially return proper rule group state if those replicas evaluated the rule group\nPRs:\nFor Rule evaluation #6129 For API HA #5773 ","categories":"","description":"","excerpt":" Author: Anand Rajagopal Date: Aug 2024 Status: Proposed Problem …","ref":"/docs/proposals/ruler-high-availability/","tags":"","title":"Ruler High Availability"},{"body":" Author: Rees Dooley Date: November 2021 Status: Accepted Overview This document aims to describe how to implement the ability to allow rules to cover data from more than a single Cortex tenant, here after referred to as federated rules. Since currently rules are owned by, query data from and save resulting series in the same tenant, this document aims to provide clear delineation of who owns a federated rule, what tenants the federated rule queries data from and where the federated rule saves resulting series.\nA federated rule is any rule which contains the src_tenants field.\nReasoning The primary use case for allowing federated rules which query data from multiple tenants is the administration of cortex.\nIn the case of the administration of cortex, when running Cortex within a large organization, there may be metrics spanning across tenants which might be desired to be monitored e.g. administrative metrics of the cortex system like prometheus_rule_evaluation_failures_total aggregated by __tenant_id__. In this case, a team e.g. infra may wish to be able to create a rule, owned by infra which queries multiple tenants t0|t1|...|ti and stores resulting series in infra.\nChallenges Allow federated rules behind feature flag Challenge Federated tenant rules and alerts will not be a good fit for organization and should be behind a feature flag.\nProposal For federated rules, creation of federated rules (those sourcing data from multiple tenants) should be blocked behind the feature flag ruler.enable-federated-rules\nIf tenant federation is enabled, then ruler should use a mergeQueryable to aggregate the results of querying multiple tenants.\nAllow federated rules only for select tenants Challenge For many organizations, the ability for any tenant to write a rule querying any other tenant is not acceptable and more fine grained control is required\nProposal Since the current default is that a tenant should only be able to write rules against itself, we suggest a config option ruler.allowed-federated-tenants, a string slice of OrgIDs like infra or 0|1|2|3|4 which are allowed to write rules against all tenants. If a tenant bar attempts to create a federated rule, an error should be returned by the ruler api. Similarly an option ruler.disallowed-federated-tenants explicitly states a list of tenants for which federated rules are not allowed. Combining these in a util.AllowedTenants should allow one to quickly determine if federation is enabled or disabled for a given tenant at rule creation.\nWhere to store resulting series of federated rule Challenge A single tenant rule always stores produced series in the tenant where the rule exists. This 1 -\u003e 1 mapping becomes a many -\u003e 1 mapping for federated rules.\nProposal Tenants owning a federated rule the resulting series is saved in the tenant which owns the rule.\nWhich tenants to query from for federated rules Challenge A single tenant rule always queries the tenant which owns the rule. This 1 -\u003e 1 mapping becomes a 1 -\u003e many mapping for federated rules.\nProposal As some use cases will demand that a specific federated rule, querying tenant B and C, is stored in the owning teams tenant A, an option to allow explicit assignment of source tenants for a federated rule is needed.\nTo support this we suggest an additional field src_tenants on the rule group containing an array of OrgIDs e.g. [t0,t1,...,ti] which when present determines which tenants to query for the given rule. Rule group is chosen as it reduces repetition between rules.\nConclusion Challenge Status Allow federated rules behind feature flag Planned but not yet implemented Allow federated rules only for select tenants Planned but not yet implemented Where to store resulting series of federated rules Planned but not yet implemented Which tenants to query from for federated rules Planned but not yet implemented ","categories":"","description":"","excerpt":" Author: Rees Dooley Date: November 2021 Status: Accepted Overview …","ref":"/docs/proposals/ruler-tenant-federation/","tags":"","title":"Ruler Tenant Federation"},{"body":" Author: Josh Abreu Date: December 2020 Status: Accepted Context and Background The Cortex Alertmanager at its current state supports high-availability by using the same technique as the upstream Prometheus Alertmanager: we gossip silences and notifications between replicas to achieve eventual consistency. This allows us to tolerate machine failure without interruption of service. The caveat here is traffic between the ruler and Alertmanagers must not be load balanced as alerts themselves are not gossiped.\nBy itself it is not horizontally scalable; it is recommended to run a maximum of 3 replicas (for high availability). Each alertmanager replica will contain an internal instance for every tenant. The Alertmanager uses roughly ~3.7MB of memory and ~0.001 CPU cores per tenant, with an average tenant having ~10 active alerts and ~1 silences. These stats were measured from a cluster with ~80 QPS (Alerts received Per Second), ~40 NPS (Notifications Per Second), and ~700 configurations over a single replica.\nProblem and Requirements Current numbers show a reasonably sized machine can handle 2000 tenants in the current service. We would like to be able to scale this up to 10x without increasing the machine size; i.e. we would like to make Cortex Alertmanager service horizontally scalable.\nFurthermore, we desire to preserve the following characteristics:\nA single Cortex Alertmanager replica crashing or exiting abruptly should not cause externally-visible downtime or failure to deliver notifications. Users should have an eventually consistent view of all the alerts current firing and under active notification, favouring availability of the Cortex Alertmanager over consistency. We should be able to scale up, down and roll out new versions of the service without any service interruption or data loss. Design This is a big design, and as such we have divided the problem up into 4 areas: Routing \u0026 Sharding, Persistence \u0026 State, Replication \u0026 Consistency, and Architecture. The options and solutions I propose are divided along these lines to aid clarity.\nRouting \u0026 Sharding To achieve horizontal scalability, we need to distribute the workload among replicas of the service. We need to choose an appropriate field to use to distribute the workload. The field must be present on all the API requests to the Alertmanager service.\nWe propose the sharding on Tenant ID. The simplicity of this implementation, would allow us to get up and running relatively quickly whilst helping us validate assumptions. We intend to use the existing ring code to manage this. Other options such as tenant ID + receiver or Tenant ID + route are relatively complex as distributor components (in this case the Ruler) would need to be aware of Alertmanager configuration.\nPersistence \u0026 State Alertmanager is a stateful service; it stores the notification state and configured silences. By default, Alertmanager persists its state to disk every 15mins. In the horizontally scalable Alertmanager service, we need to move this state around as the number of replicas grows and shrinks. We also need to persist this state across restarts and rolling upgrades.\nWe propose making each Alertmanager replica flush the state to object storage, under its own key that’s a combination of tenant ID + replica periodically. This state on durable storage will only be used when cold-starting the cluster.\nThis mechanism covers multiple challenges (scaling up \u0026 down, rolling restarts, total cluster failure). To top this off, in the implementation, we’ll always try to request state from other replicas in the cluster before ever trying to go to object storage.\nReplication \u0026 Consistency Upstream Alertmanager replicates notification state between replicas to ensure notifications are not sent more than once. Cortex Alertmanager does the same. When we move to a model where AM instances for a given tenant only live on a subset of replicas, we have to decide how we will keep these replicas in sync.\nWe have an option of doing nothing and use the existing gossip mechanism and gossip all tenants state to all replicas. The AM Router will then drop state updates for tenants which don’t shard to a given replica. I think this will be easy to implement as it requires few changes but probably won’t scale.\nHowever, I propose we Synchronized state over gRPC. The upstream Alertmanager notification dispatcher uses a timeout-based approach for its notifications. Using the “peer order”, it’ll wait a certain amount of time before letting other replicas know if this notification succeeded or not. If it didn’t, the next replica in line will try to send the notification. We propose to communicate this via gRPC calls.\nWhile I propose the use of gRPC, is good to note that doing nothing is still a solid candidate for consideration but not having to maintain and operate two separate replication patterns feels like a win.\nArchitecture To implement the sharding strategy we need a component in charge of handling incoming alerts and API requests, distributing it to the corresponding shard.\nThe idea here is to have an “Alertmanager Distributor” as a first stop in the reception of alerts. Once alerts are received, the component is in charge of validating the alerts against the limits. Validated alerts are then sent to multiple managers in parallel.\nThe individual pieces of this component (sharding, limits) cannot be optional - the optional part of it is where we decide to run it.\nWe can either run it as a separate service or embed it. I propose we simply embed it. At its core it’ll be simpler to operate. With future work making it possible to run as a separate service so that operators can scale when/if needed.\nConclusion Under the assumption we implement the options proposed above, our architecture looks like this:\nPOST /api/v1/alerts (from the ruler) can go to any Alertmanager replica. The AM distributor uses the ring to write alerts to a quorum of AM managers (reusing the existing code). We continue to use the same in-memory data structure from the upstream Alertmanager to save alerts and notify other pieces\nGET /api/v1/alerts and /api/v1/alerts/group uses the ring to find the right alertmanager replicas for the given tenant. They read from a quorum of alertmanager replicas and return an union of the results.\nAlertmanager state is replicated between instances to keep them in sync. Where the state is replicated to to is controlled by the ring.\n","categories":"","description":"","excerpt":" Author: Josh Abreu Date: December 2020 Status: Accepted Context and …","ref":"/docs/proposals/scalable-alertmanager/","tags":"","title":"Scalable Alertmanager"},{"body":" Author: Joe Elliott Date: April 2020 Status: Proposed Overview This document aims to describe the role that the Cortex Query Frontend plays in running multitenant Cortex at scale. It also describes the challenges of horizontally scaling the query frontend component and includes several recommendations and options for creating a reliably scalable query-frontend. Finally, we conclude with a discussion of the overall philosophy of the changes and propose an alternative.\nFor the original design behind the query frontend, you should read Cortex Query Optimisations design doc from 2018-07.\nReasoning Query frontend scaling is becoming increasingly important for two primary reasons.\nThe Cortex team is working toward a scalable single binary solution. Recently the query-frontend was added to the Cortex single binary mode and, therefore, needs to seamlessly scale. Technically, nothing immediately breaks when scaling the query-frontend, but there are a number of concerns detailed in Challenges And Proposals.\nAs the query-frontend continues to support additional features it will start to become a bottleneck of the system. Current wisdom is to run very few query-frontends in order to maximize Tenancy Fairness but as more features are added scaling horizontally will become necessary.\nQuery Frontend Role Load Shedding The query frontend maintains a queue per tenant of configurable length (default 100) in which it stores a series of requests from that tenant. If this queue fills up then the frontend will return 429’s thus load shedding the rest of the system.\nThis is particularly effective due to the “pull” based model in which queriers pull requests from query frontends.\nQuery Retries The query frontend is capable of retrying a query on another querier if the first should fail due to OOM or network issues.\nSharding/Parallelization The query frontend shards requests by interval and other factors to concurrently run a single query across multiple queriers.\nQuery Alignment/Caching Queries are aligned to their own step and then stored/retrieved from cache.\nTenancy Fairness By maintaining one queue per tenant, a low demand tenant will have the same opportunity to have a query serviced as a high demand tenant. See Dilutes Tenant Fairness for additional discussion.\nFor clarity, tenancy fairness only comes into play when queries are actually being queued in the query frontend. Currently this rarely occurs, but as query sharding becomes more aggressive this may become the norm.\nChallenges And Proposals Dynamic Querier Concurrency Challenge For every query frontend the querier adds a configurable number of goroutines which are each capable of executing a query. Therefore, scaling the query frontend impacts the amount of work each individual querier is attempting to do at any given time.\nScaling up may cause a querier to attempt more work than they are configured for due to restrictions such as memory and cpu limits. Additionally, the promql engine itself is limited in the number of queries it can do as configured by the -querier.max-concurrent parameter. Attempting more queries concurrently than this value causes the queries to queue up in the querier itself.\nFor similar reasons scaling down the query frontend may cause a querier to not use its allocated memory and cpu effectively. This will lower effective resource utilization. Also, because individual queriers will be doing less work, this may cause increased queueing in the query frontends.\nProposal Currently queriers are configured to have a max parallelism per query frontend. An additional “total max concurrency” flag should be added.\nTotal Max Concurrency would then be evenly divided amongst all available query frontends. This would decouple the amount of work a querier is attempting to do with the number of query frontends that happen to exist at this moment. Consequently this would allow allocated resources (e.g. k8s cpu/memory limits) to remain balanced with the work the querier was attempting as the query frontend is scaled up or down.\nA PR has already been merged to address this.\nOverwhelming PromQL Concurrency Challenge If #frontends \u003e promql concurrency then the queriers are incapable of devoting even a single worker to each query frontend without risking queueing in the querier. Queuing in the querier is a highly undesirable state and one of the primary reasons the query frontend was originally created.\nProposal When #frontends \u003e promql concurrency then each querier will maintain exactly one connection to every frontend. As the query frontend is currently coded it will attempt to use every open GRPC connection to execute a query in the attached queriers. Therefore, in this situation where #frontends \u003e promql concurrency, the querier is exposing itself to more work then it is actually configured to perform.\nTo prevent this we will add “flow control” information to the ProcessResponse message that is used to return query results from the querier to the query frontend. In an active system this message is passed multiple times per second from the queriers to the query frontends and would be a reliable way for the frontends to track the state of queriers and balance load.\nThere are a lot of options for an exact implementation of this idea. An effective solution should be determined and chosen by modeling a set of alternatives. The details of this would be included in another design doc. A simple implementation would look something like the following:\nAdd two new fields to ProcessResponse:\nmessage ProcessResponse { httpgrpc.HTTPResponse httpResponse = 1; currentConcurrency int = 2; desiredConcurrency int = 3; } currentConcurrency - The current number of queries being executed by the querier.\ndesiredConcurrency - The total number of queries that a querier is capable of executing.\nAdd a short backoff to the main frontend processing loop. This would cause the frontend to briefly back off of any querier that was overloaded but continue to send queries to those that were capable of doing work.\nif current \u003e desired { zzz := (current - desired) * backoffDuration zzz *= 1 + rand.Float64() * .1 // jitter time.Sleep(zzz) } Passing flow control information from the querier to the frontend would also open up additional future work for more sophisticated load balancing across queriers. For example by simply comparing and choosing the least congested of two queriers we could dramatically improve how well work is distributed.\nIncreased Time To Failure Challenge Scaling the query frontend also increases the per tenant queue length by creating more queues. This could result in increased latencies where failing fast (429) would have been preferred.\nThe operator could reduce the queue length per query frontend in response to scaling out, but then they would run the risk of unnecessarily failing a request due to unbalanced distribution across query frontends. Also, shorter queues run the risk of failing to properly service heavily sharded queries.\nAnother concern is that a system with more queues will take longer to recover from an production event as it will have queued up more work.\nProposal Currently we are not proposing any changes to alleviate this concern. We believe this is solvable operationally. This can be revisited as more information is gathered.\nQuerier Discovery Lag Challenge Queriers have a configurable parameter that controls how often they refresh their query frontend list. The default value is 10 seconds. After a new query frontend is added the average querier will take 5 seconds (after DNS is updated) to become aware of it and begin requesting queries from it.\nProposal It is recommended to add a readiness/health check to the query frontend to prevent it from receiving queries while it is waiting for queriers to connect. HTTP health checks are supported by envoy, k8s, nginx, and basically any commodity load balancer. The query frontend would not indicate healthy on its health check until at least one querier had connected.\nIn a k8s environment this will require two services. One service for discovery with publishNotReadyAddresses set to true and one service for load balancing which honors the healthcheck/readiness probe. After a new query-frontend instance is created the “discovery service” would immediately have the ip of the new instance which would allow queriers to discover and attach to it. After queriers had connected it would then raise its readiness probe and appear on the “load balancing” service and begin receiving traffic.\nDilutes Tenant Fairness Challenge Given f query frontends, n tenants and an average of q queries in the frontend per tenant. The following assumes that queries are perfectly distributed across query frontends. The number of tenants per instance would be:\nThe chance that a query by a tenant with Q queries in the frontend is serviced next is:\nNote that fewer query frontends caps the impact of the number of active queries per tenant. If there is only one query frontend then the equation reduces to:\nand every tenant has an equal chance of being serviced regardless of the number of queued queries.\nAdding more query frontends favors high volume tenants by giving them more slots to be picked up by the next available querier. Fewer query frontends allows for an even playing field regardless of the number of active queries.\nFor clarity, it should be noted that tenant fairness is only impacted if queries are being queued in the frontend. Under normal operations this is currently not occurring although this may change with increased sharding.\nProposal Tenancy fairness is complex and is currently not impacting our system. Therefore we are proposing a very simple improvement to the query frontend. If/when frontend queuing becomes more common this can be revisited as we will understand the problem better.\nCurrently the query frontend picks a random tenant to service when a querier requests a new query. This can increase long tail latency if a tenant gets “unlucky” and is also exacerbated for low volume tenants by scaling the query frontend. Instead the query frontend could use a round robin approach to choose the next tenant to service. Round robin is a commonly used algorithm to increase fairness in scheduling.\nThis would be a very minor improvement, but would give some guarantees to low volume tenants that their queries would be serviced. This has been proposed in this issue.\nPros: Requires local knowledge only. Easier to implement than weighted round robin.\nCons: Improvement is minor.\nAlternatives to Round Robin\nDo Nothing\nAs is noted above tenancy fairness only comes into play when queries start queueing up in the query frontend. Internal Metrics for multi-tenant Cortex at Grafana show that this has only happened 5 times in the past week significantly enough to have been caught by Prometheus.\nRight now doing nothing is a viable option that will, almost always, fairly serve our tenants. There is, however, some concern that as sharding becomes more commonplace queueing will become more common and QOS will suffer due to reasons outlined in Dilutes Tenant Fairness.\nPros: Easy!\nCons: Nothing happens!\nWeighted Round Robin\nThe query frontends could maintain a local record of throughput or work per tenant. Tenants could then be sorted in QOS bands. In its simplest form there would be two QOS bands. The band of low volume tenants would be serviced twice for every one time the band of high volume tenants would be serviced. The full details of this approach would require a separate proposal.\nThis solution would also open up interesting future work. For instance, we could allow operators to manually configure tenants into QOS bands.\nPros: Requires local knowledge only. Can be extended later to allow tenants to be manually sorted into QOS tiers.\nCons: Improvement is better than Round Robin only. Relies on even distribution of queries across frontends. Increased complexity and difficulty in reasoning about edge cases.\nWeighted Round Robin With Gossiped Traffic\nThis approach would be equivalent to Weighted Round Robin proposed above but with tenant traffic volume gossiped between query frontends.\nPros: Benefits of Weighted Round Robin without the requirement of even query distribution. Even though it requires distributed information a failure in gossip means it gracefully degrades to Weighted Round Robin.\nCons: Requires cross instance communication. Increased complexity and difficulty in reasoning about edge cases.\nAlternative The proposals in this document have preferred augmenting existing components to make decisions with local knowledge. The unstated goal of these proposals is to build a distributed queue across a scaled query frontend that reliably and fairly serves our tenants.\nOverall, these proposals will create a robust system that is resistant to network partitions and failures of individual pieces. However, it will also create a complex system that could be difficult to reason about, contain hard to ascertain edge cases and nuanced failure modes.\nThe alternative is, instead of building a distributed queue, to add a new cortex queueing service that sits in between the frontends and the queriers. This queueing service would pull from the frontends and distribute to the queriers. It would decouple the stateful queue from the stateless elements of the query frontend and allow us to easily scale the query frontend while keeping the queue itself a singleton. In a single binary HA mode one (or few) of the replicas would be leader elected to serve this role.\nHaving a singleton queue is attractive because it is simple to reason about and gives us a single place to make fair cross tenant queueing decisions. It does, however, create a single point of failure and add another network hop to the query path.\nConclusion In this document we reviewed the reasons the frontend exists, challenges and proposals to scaling the frontend and an alternative architecture that avoids most problems but comes with its own challenges.\nChallenge Proposal Status Dynamic Querier Concurrency Add Max Total Concurrency in Querier Pull Request Overwhelming PromQL Concurrency Queriers Coordinate Concurrency with Frontends Proposed Increased Time to Failure Operational/Configuration Issue. No Changes Proposed. N/A Querier Discovery Lag Query Frontend HTTP Health Checks Pull Request Dilutes Tenant Fairness Round Robin with additional alternatives proposed Pull Request ","categories":"","description":"","excerpt":" Author: Joe Elliott Date: April 2020 Status: Proposed Overview This …","ref":"/docs/proposals/scalable-query-frontend/","tags":"","title":"Scalable Query Frontend"},{"body":" Author: @pracucci, @tomwilkie, @pstibrany Reviewers: Date: August 2020 Status: Accepted, implemented in PR #3090 Shuffle sharding and zone awareness Background Cortex shards the received series across all available ingesters. In a multi-tenant cluster, each tenant series are sharded across all ingesters. This allows to horizontally scale the series across the pool of ingesters but also suffers some issues:\nGiven every tenant writes series to all ingesters, there’s no isolation between tenants - a single misbehaving tenant can affect the whole cluster. Each ingester needs an open TSDB per tenant per ingester - which has significant memory overhead. The larger the number of tenants, the higher the TSDB memory overhead, regardless of the number of series stored in each TSDB. Similarly, the number of uploaded blocks to the storage every 2 hours is a function of the number of TSDBs open for each ingester. A cluster with a large number of small tenants will upload a very large number of blocks to the storage, each block being very small, increasing the number of API calls against the storage bucket. Cortex currently supports sharding a tenant to a subset of the ingesters on the write path PR, using a feature called “subring”. However, the current subring implementation suffers two issues:\nNo zone awareness: it doesn’t guarantee selected instances are balanced across availability zones No shuffling: the implementation is based on the hash ring and it selects N consecutive instances in the ring. This means that, instead of minimizing the likelihood that two tenants share the same instances, it emphasises it. In order to provide a good isolation between tenants, we want to minimize the chances that two tenants share the same instances. Goal The goal of this work is to fix “shuffling” and “zone-awareness” when building the subring for a given tenant, honoring the following properties:\nStability: given the same ring, the algorithm always generates the same subring for a given tenant, even across different machines Consistency: when the ring is resized, only n/m series are remapped on average (where n is the number of series and m is the number of replicas). Shuffling: probabilistically and for a large enough cluster, ensure every tenant gets a different set of instances, with a reduced number of overlapping instances between two tenants to improve failure isolation. Zone-awareness (balanced): the subring built for each tenant contains a balanced number of instances for each availability zone. Selecting the same number of instances in each zone is an important property because we want to preserve the balance of in-memory series across ingesters. Having less replicas in one zone will mean more load per node in this zone, which is something we want to avoid. Proposal This proposal is based on Amazon’s Shuffle Sharding article and the algorithm has been inspired by shuffle sharding implementation in the AWS Route53 infima library.\nGiven a tenant and a shard size S (number of instances to which tenant data/workload should be sharded to), we build a subring selecting N instances from each zone, where N = ceil(S / num of zones). The shard size S is required to be a multiple of the number of zones, in order to select an equal number of instances from each zone.\nTo do it, we treat each zone as a separate ring and select N unique instances from each zone. The instances selection process works as follow:\nGenerate a seed based on the tenant ID Initialise a pseudo random number generator with the tenant’s seed. The random generator must guarantee predictable numbers given the same input seed. Generate a sequence of N random numbers, where N is the number of instances to select from the zone. Each random number is used as a “token” to look up instances in the ring. For each random number: Lookup the instance holding that token in the ring If the instance has not been previously selected, then pick it If the instance was previously selected (we call this a “collision”), then continue walking the ring clockwise until we find an instance which has not been selected yet Guaranteed properties Stability The same tenant ID always generates the same seed. Given the same seed, the pseudo number random generator always generates the same sequence of numbers.\nThis guarantees that, given the same ring, we generate the same exact subring for a given tenant.\nConsistency The consistency property is honored by two aspects of the algorithm:\nThe quantity of random numbers generated is always equal to the shard size S, even in case of “collisions”. A collision is when the instance holding the random token has already been picked and we need to select a different instance which has not been picked yet. In case of collisions, we select the “next” instance continuing walking the ring instead of generating another random number Example adding an instance to the ring Let’s consider an initial ring with 3 instances and 1 zone (for simplicity):\nI1 - Tokens: 1, 8, 15 I2 - Tokens: 5, 11, 19 I3 - Tokens: 7, 13, 21 With a replication factor = 2, the random sequence looks up:\n3 (I2) 6 (I1) Then we add a new instance and the updated ring is:\nI1 - Tokens: 1, 8, 15 I2 - Tokens: 5, 11, 19 I3 - Tokens: 7, 13, 21 I4 - Tokens: 4, 7, 17 Now, let’s compare two different algorithms to solve collisions:\nUsing the random generator:\nRandom sequence = 3 (I4), 6 (I4 - collision), 12 (I3)\nall instances are different (I4, I3) Walking the ring:\nRandom sequence = 3 (I4), 6 (I4 - collision, next is I1)\nonly 1 instance is different (I4, I1) Shuffling Unless when resolving collisions, the algorithm doesn’t walk the ring to find the next instances, but uses a sequence of random numbers. This guarantees instances are shuffled, between different tenants, when building the subring.\nZone-awareness We treat each zone as a separate ring and select an equal number of instances from each zone. This guarantees a fair balance of instances between zones.\nProof of concept We’ve built a reference implementation of the proposed algorithm, to test the properties described above.\nIn particular, we’ve observed that the actual distribution of matching instances between different tenants is very close to the theoretical one, as well as consistency and stability properties are both honored.\n","categories":"","description":"","excerpt":" Author: @pracucci, @tomwilkie, @pstibrany Reviewers: Date: August …","ref":"/docs/proposals/shuffle-sharding-and-zone-awareness/","tags":"","title":"Shuffle sharding and zone awareness"},{"body":" Author: @pracucci, @tomwilkie, @pstibrany Reviewers: Date: August 2020 Status: Proposed, partially implemented Background Cortex currently supports sharding of tenants to a subset of the ingesters on the write path PR.\nThis feature is called “subring”, because it computes a subset of nodes registered to the hash ring. The aim of this feature is to improve isolation between tenants and reduce the number of tenants impacted by an outage.\nThis approach is similar to the techniques described in Amazon’s Shuffle Sharding article, but currently suffers from a non random selection of nodes (proposed solution below).\nCortex can be configured with a default subring size, and then it can be customized on a per-tenant basis. The per-tenant configuration is live reloaded during runtime and applied without restarting the Cortex process.\nThe subring sharding currently supports only the write-path. The read-path is not shuffle sharding aware. For example, an outage of more than one ingester with RF=3 will affect all tenants, or a particularly noisy tenant wrt queries has the ability to affect all tenants.\nGoals The Cortex read path should support shuffle sharding to isolate the impact of an outage in the cluster. The shard size must be dynamically configurable on a per-tenant basis during runtime.\nThis deliverable involves introducing shuffle sharding in:\nQuery-frontend → Querier (for queries sharding) PR #3113 Querier → Store-gateway (for blocks sharding) PR #3069 Querier→ Ingesters (for queries on recent data) Ruler (for rule and alert evaluation) Prerequisite: fix subring shuffling The solution is implemented in https://github.com/cortexproject/cortex/pull/3090.\nThe problem The subring is a subset of nodes that should be used for a specific tenant.\nThe current subring implementation doesn’t shuffle tenants across nodes. Given a tenant ID, it finds the first node owning the hash(tenant ID) token and then it picks N distinct consecutive nodes walking the ring clockwise.\nFor example, in a cluster with 6 nodes (numbered 1-6) and a replication factor of 3, three tenants (A, B, C) could have the following shards:\nTenant ID Node 1 Node 2 Node 3 Node 4 Node 5 Node 6 A x x x B x x x C x x x Proposal We propose to build the subring picking N distinct and random nodes registered in the ring, using the following algorithm:\nSID = tenant ID SID = hash(SID) Look for the node owning the token range containing FNV-1a(SID) Loop to (2) until we’ve found N distinct nodes (where N is the shard size) hash() function to be decided. The required property is to be strong enough to not generate loops across multiple subsequent hashing of the previous hash.\nQuery-frontend → Queriers shuffle sharding Implemented in https://github.com/cortexproject/cortex/pull/3113.\nHow querier runs query-frontend jobs Today each querier connects to each query-frontend instance, and calls a single “Process” method via gRPC.\n“Process” is a bi-directional streaming gRPC method – using the server-to-client stream for sending requests from query-frontend to the querier, and client-to-server stream for returning results from querier to the query-frontend. NB this is the opposite of what might be considered normal. Query-frontend scans all its queues with pending query requests, and picks a query to execute based on a fair schedule between tenants.\nThe query request is then sent to an idle querier worker over the stream opened in the Process method, and the query-frontend then waits for a response from querier. This loop repeats until querier disconnects.\nProposal To support shuffle sharding, Query-Frontends will keep a list of connected Queriers, and randomly (but consistently between query-frontends) choose N of them to distribute requests to. When Query-Frontend looks for the next request to send to a given querier, it will only consider tenants that “belong” to the Querier.\nTo choose N Queriers for a tenant, we propose to use a simple algorithm:\nSort all Queriers by their ID SID = tenant ID SID = hash(SID) Pick the querier from the list of sorted queries with:\nindex = FNV-1a(SID) % number of Queriers Loop to (3) until we’ve found N distinct queriers (where N is the shard size) and stop early if there aren’t enough queriers hash() function to be decided. The required property is to be strong enough to not generate loops across multiple subsequent hashing of the previous hash.\nProperties Stability: this will produce the same result on all query-frontends as long as all queriers are connected to all query-frontends. Simplicity: no external dependencies. No consistent hashing: adding/removing queriers will cause “resharding” of tenants between queriers. While in general that’s not desirable property, queriers are stateless so it doesn’t seem to matter in this case. Implementation notes Caching: once this list of queriers to use for a tenant is computed in the query-frontend, it is cached in memory until queriers are added or removed. Per-tenant cache entries will have a TTL to discard tenants not “seen” since a while. Querier ID: Query-frontends currently don’t have any identity for queriers. We need to introduce sending of a unique ID (eg. hostname) by querier to query-frontend when it calls “Process” method. Backward-compatibility: when querier shuffle sharding is enabled, the system expects that both query-frontend and querier will run a compatible version. Cluster version upgrade will require to rollout new query-frontends and queriers first, and then enable shuffle sharding. UI: we propose to expose the current state of the query-frontend through a new endpoint which should display: Which querier are connected to the query-frontend Are there any “old” queriers, that are receiving requests from all tenants? Mapping of tenants to queriers. Note that this mapping may only be available for tenants with pending requests on given query-frontend, and therefore be very dynamic. Configuration Shard size will be configurable on a per-tenant basis via existing “runtime-configuration” mechanism (limits overrides). Changing a value for a tenant needs to invalidate cached per-tenant queriers. Queriers shard size will be a different setting than then one used for writes. Evaluated alternatives Use the subring An alternative option would be using the subring. This implies having queriers registering to the hash ring and query-frontend instances using the ring client to find the queriers subring for each tenant.\nThis solution looks adding more complexity without any actual benefit.\nChange query-frontend → querier architecture Completely different approach would be to introduce a place where starting queriers would register (eg. DNS-based service discovery), and let query-frontends discover queriers from this central registry.\nPossible benefit would be that queriers don’t need to initiate connection to all query-frontends, but query-frontends would only connect to queriers for which they have actual pending requests. However this would be a significant redesign of how query-frontend / querier communication works.\nQuerier → Store-gateway shuffle sharding Implemented in https://github.com/cortexproject/cortex/pull/3069.\nIntroduction As of today, the store-gateway supports blocks sharding with customizable replication factor (defaults to 3). Blocks of a single tenant are sharded across all store-gateway instances and so to execute a query the querier may touch any store-gateway in the cluster.\nThe current sharding implementation is based on a hash ring formed by store-gateway instances.\nProposal The proposed solution to add shuffle sharding support to the store-gateway is to leverage on the existing hash ring to build a per-tenant subring, which is then used both by the querier and store-gateway to know to which store-gateway a block belongs to.\nConfiguration Shuffle sharding can be enabled in the store-gateway configuration. It supports a default sharding factor, which is overridable on a per-tenant basis and live reloaded during runtime (using the existing limits config). The querier already requires the store-gateway configuration when the blocks sharding is enabled. Similarly, when shuffle sharding is enabled the querier will require the store-gateway shuffle sharding configuration as well. Implementation notes When shuffle sharding is enabled:\nThe store-gateway syncUsersBlocks() will build a tenant’s subring for each tenant found scanning the bucket and will skip any tenant not belonging to its shard.\nLikewise, ShardingMetadataFilter will first build a tenant’s subring and then will use the existing logic to filter out blocks not belonging to store-gateway instance itself. The tenant ID can be read from the block’s meta.json. The querier blocksStoreReplicationSet.GetClientsFor() will first build a tenant’s subring and then will use the existing logic to find out to which store-gateway instance each requested block belongs to. Evaluated alternatives Given the store-gateways already form a ring and building the shuffle sharding based on the ring (like in the write path) doesn’t introduce extra operational complexity, we haven’t discussed alternatives.\nQuerier→ Ingesters shuffle sharding We’re currently discussing/evaluating different options.\nProblem Cortex must guarantee query correctness; transiently incorrect results may be cached and returned forever. The main problem to solve when introducing ingesters shuffle sharding on the read path is to make sure that a querier fetch data from all ingesters having at least 1 sample for a given tenant.\nThe problem to solve is: how can a querier efficiently find which ingesters have data for a given tenant? Each option must consider the changing of the set of ingesters and the changing of each tenant’s subring size.\nProposal: use only the information contained in the ring. This section describes an alternative approach. Discussion is still on-going.\nThe idea is for the queries to be able to deduce what ingesters could possibly hold data for a given tenant by just consulting the ring (and the per-tenant sub ring sizes). We posit that this is possible with only a single piece of extra information: a single timestamp per ingester saying when the ingester first joined the ring.\nScenario: ingester scale up When a new ingester is added to the ring, there will be a set of user subrings that see a change: an ingester being removed, and a new one being added. We need to guarantee that for some time period (the block flush interval), the ingester removed is also consulted for queries.\nTo do this, during the subring selection if we encounters an ingester added within the time period, we will add this to the subring but continue node selection as before - in effect, selecting an extra ingester:\nvar ( subringSize int selectedNodes []Node deadline = time.Now().Add(-flushWindow) ) for len(selectedNodes) \u003c subringSize { token := random.Next() node := getNodeByToken(token) for { if node in selectedNodes { node = node.Next() continue } if node.Added.After(deadline) { subringSize++ selectedNodes.Add(node) node = node.Next() continue ) selectedNodes.Add(node) break ) } Scenario: ingester scale down When an ingester is permanently removed from the ring it will flush its data to the object store and the subrings containing the removed ingester will gain a “new” ingester. Queries consult the store and merge the results with those from the ingesters, so no data will be missed.\nQueriers and store-gateways will discover newly flushed blocks on next sync (-blocks-storage.bucket-store.sync-interval, default 5 minutes). Multiple ingesters should not be scaled-down within this interval.\nTo improve read-performance, queriers and rulers are usually configured with non-zero value of -querier.query-store-after option. This option makes queriers and rulers to consult only ingesters when running queries within specified time window (eg. 12h). During scale-down this needs to be lowered in order to let queriers and rulers use flushed blocks from the storage.\nScenario: increase size of a tenant’s subring Node selection for subrings is stable - increasing the size of a subring is guaranteed to only add new nodes to it (and not remove any nodes). Hence, if a tenant’s subring is increase in size the queriers will notice the config change and start consulting the new ingester.\nScenario: decreasing size of a tenant’s subring If a tenant’s subring decreases in size, there is currently no way for the queriers to know how big the ring was previously, and hence they will potentially miss an ingester with data for that tenant.\nThis is deemed an infrequent operation that we considered banning, but have a proposal for how we might make it possible:\nThe proposal is to have separate read subring and write subring size in the config. The read subring will not be allowed to be smaller than the write subring. When reducing the size of a tenant’s subring, operators must first reduce the write subring, and then two hours later when the blocks have been flushed, the read subring. In the majority of cases the read subring will not need to be specified, as it will default to the write subring size.\nConsidered alternative #1: Ingesters expose list of tenants A possible solution could be keeping in the querier an in-memory data structure to map each ingester to the list of tenants for which it has some data. This data structure would be constructed at querier startup, and then periodically updated, interpolating two information:\nThe current state of the ring The list of tenants directly exposed by each ingester (via a dedicated gRPC call) Scenario: new querier starts up When a querier starts up and before getting ready:\nIt scans all ingesters (discovered via the ring) and fetches the list of tenants for which each ingester has some data For each found tenant (unique list of tenant IDs across all ingesters responses), the querier looks at the current state of the ring and adds to the map the list of ingesters currently assigned to the tenant shard, even if they don’t hold any data yet (because may start receiving series shortly) Then the querier watches the ingester ring and rebuilds the in-memory map whenever the ring topology changes.\nScenario: querier receives a query for an unknown tenant A new tenant starts remote writing to the cluster. The querier doesn’t know it in its in-memory map, so it adds the tenant on the fly to the map just looking at the current state of the ring.\nScenario: ingester scale up / down When a new ingester is added / removed to / from the ring, the ring topology changes and queriers will update the in-memory map.\nScenario: per-tenant shard size increases Queriers periodically (every 1m) reload the limits config file. When a tenant shard size change is detected, the querier updates the in-memory map for the affected tenant.\nIssue: some time series data may be missing in queries up to 1m.\nEdge case: queriers notice the ring topology change before distributors Consider the following scenario:\nTenant A shard is composed by ingesters 1,2,3,4,5,6 Tenant A is remote writing 1 single series and gets replicated to ingester 1,2,3 The ring topology changes and tenant A shard is ingesters 1,2,3,7,8,9 Querier notices the ring topology change and updates the in-memory map. Given tenant A series were only on ingester 1,2,3, the querier maps tenant A to ingester 1,2,3 (because of what received from ingesters via gRPC) and 7,8,9 (because of the current state of the ring) Distributor hasn’t updated the ring state yet Tenant A remote writes 1 new series, which get replicated to 4,5,6 Distributor updates the ring state Race condition: querier will not know that ingesters 4,5,6 contains tenant A data until the next sync Considered alternative #2: streaming updates from ingesters to queriers This section describes an alternative approach.\nCurrent state As of today, queriers discover ingesters via the ring:\nIngesters register (and update their heartbeat timestamp) to the ring and queriers watch the ring, keeping an in-memory copy of the latest ingesters ring state. Queriers use the in-memory ring state to discover all ingesters that should be queried at query time. Proposal The proposal is to expose a new gRPC endpoint on ingesters, which allows queriers to receive a stream of real time updates from ingesters about the tenants for which an ingester currently has time series data.\nFrom the querier side:\nAt startup the querier discovers all existing ingesters. For each ingester, the querier calls the ingester’s gRPC endpoint WatchTenants() (to be created). As soon as the WatchTenants() rpc is called, the ingester sends the entire set of tenants to the querier and then will send incremental updates (tenant added or removed from ingester) while the WatchTenants() stream connection is alive. If the querier loses the connection to an ingester, it will automatically retry (with backoff) while the ingester is within the ring. The querier watches the ring to discover added/removed ingesters. When an ingester is added, the querier adds the ingester to the pool of ingesters whose state should be monitored via WatchTenants(). At query time, the querier looks for all ingesters within the ring. There are two options: The querier knows the state of the ingester: the ingester will be queried only if it contains data for the query’s tenant. The querier doesn’t know the state of the ingester (eg. because it was just registered to the ring and WatchTenants() hasn’t succeeded yet): the ingester will be queried anyway (correctness first). The querier will fine tune gRPC keepalive settings to ensure a lost connection between the querier and ingester will be early detected and retried. Trade-offs Pros:\nThe querier logic, used to find ingesters for a tenant’s shard, does not require to watch the overrides config file (containing tenant shard size override). Watching the file in the querier is problematic because of introduced delays (ConfigMap update and Cortex file polling) which could lead to distributors apply changes before queriers. The querier never uses the current state of the ring as a source of information to detect which ingesters have data for a specific tenant. This information comes directly from the ingesters themselves, which makes the implementation less likely to be subject to race conditions. Cons:\nEach querier needs to open a gRPC connection to each ingester. Given gRPC supports multiplexing, the underlying TCP connection could be the same connection used to fetch samples from ingesters at query time, basically having 1 single TCP connection between a querier and an ingester. The “Edge case: queriers notice the ring topology change before distributors” described in attempt #1 can still happen in case of delays in the propagation of the state update from an ingester to queriers: Short delay: a short delay (few seconds) shouldn’t be a real problem. From the final user perspective, there’s no real difference between this edge case and a delay of few seconds in the ingestion path (eg. Prometheus remote write lagging behind few seconds). In the real case of Prometheus remote writing to Cortex, there’s no easy way to know if the latest samples are missing because has not been remote written yet by Prometheus or any delay in the propagation of this information between ingesters and queriers. Long delay: in case of networking issue propagating the state update from an ingester to the querier, the gRPC keepalive will trigger (because of failed ping-pong) and the querier will remove the failing ingesters in-memory data, so the ingester will be always tried by the querier for any query, until the state update will be re-established. Ruler sharding Introduction The ruler currently supports rule groups sharding across a pool of rulers. When sharding is enabled, rulers form a hash ring and each ruler uses the ring to check if it should evaluate a specific rule group.\nAt a polling interval (defaults to 1 minute), the ruler:\nList all the bucket objects to find all rule groups (listing is done specifying an empty delimiter so it return objects at any depth) For each discovered rule group, the ruler hashes the object key and checks if it belongs to the range of tokens assigned to the ruler itself. If not, the rule group is discarded, otherwise it’s kept for evaluation. Proposal We propose to introduce shuffle sharding in the ruler as well, leveraging on the already existing hash ring used by the current sharding implementation.\nThe configuration will be extended to allow to configure:\nEnable/disable shuffle sharding Default shard size Per-tenant overrides (reloaded at runtime) When shuffle sharding is enabled:\nThe ruler lists (ListBucketV2) the tenants for which rule groups are stored in the bucket The ruler filters out tenants not belonging to its shard For each tenant belonging to its shard, the ruler does a ListBucketV2 call with the “/” prefix and with empty delimiter to find all the rule groups, which are then evaluated in the ruler The ruler re-syncs the rule groups from the bucket whenever one of the following conditions happen:\nPeriodic interval (configurable) Ring topology changes The configured shard size of a tenant has changed Other notes The “subring” implementation is unoptimized. We will optimize it as part of this work to make sure no performance degradation is introduced when using the subring vs the normal ring. ","categories":"","description":"","excerpt":" Author: @pracucci, @tomwilkie, @pstibrany Reviewers: Date: August …","ref":"/docs/proposals/shuffle-sharding-on-the-read-path/","tags":"","title":"Shuffle sharding on the read path"},{"body":" Author: @gotjosh Reviewers: @gouthamve, @pracucci Date: March 2020 Status: Accepted Problem Statement Prometheus holds metric metadata alongside the contents of a scrape. This metadata (HELP, TYPE, UNIT and METRIC_NAME) enables some Prometheus API endpoints to output the metadata for integrations (e.g. Grafana) to consume it.\nAt the moment of writing, Cortex does not support the api/v1/metadata endpoint that Prometheus implements as metadata was never propagated via remote write. Recent work is done in Prometheus enables the propagation of metadata.\nWith this in place, remote write integrations such as Cortex can now receive this data and implement the API endpoint. This results in Cortex users being able to enjoy a tiny bit more insight on their metrics.\nPotential Solutions Before we delve into the solutions, let’s set a baseline about how the data is received. This applies almost equally for the two.\nMetadata from Prometheus is sent in the same WriteRequest proto message that the samples use. It is part of a different field (#3 given #2 is already used interally), the data is a set identified by the metric name - that means it is aggregated across targets, and is sent all at once. Implying, Cortex will receive a single WriteRequest containing a set of the metadata for that instance at an specified interval.\n. It is also important to note that this current process is an intermediary step. Eventually, metadata in a request will be sent alongside samples and only for those included. The solutions proposed, take this nuance into account to avoid coupling between the current and future state of Prometheus, and hopefully do something now that also works for the future.\nAs a reference, these are some key numbers regarding the size (and send timings) of the data at hand from our clusters at Grafana Labs:\nOn average, metadata (a combination of HELP, TYPE, UNIT and METRIC_NAME) is ~55 bytes uncompressed. at GL, on an instance with about 2.6M active series, we hold ~1241 unique metrics in total. with that, we can assume that on a worst-case scenario the metadata set for that instance is ~68 kilobytes uncompressed. by default, this data is only propagated once every minute (aligning with the default scrape interval), but this can be adjusted. Finally, what this gives us is a baseline worst-case scenario formula for the data to store per tenant: ~68KB * Replication Factor * # of Instances. Keeping in mind that typically, there’s a very high overlap of metadata across instances, and we plan to deduplicate in the ingesters. Write Path Store the metadata directly from the distributors into a cache (e.g. Memcached) Since metadata is received all at once, we could directly store into an external cache using the tenant ID as a key, and still, avoid a read-modify-write. However, a very common use case of Cortex is to have multiple Prometheus sending data for the same tenant ID. This complicates things, as it adds a need to have an intermediary merging phase and thus making a read-modify-write inevitable.\nKeep metadata in memory within the ingesters Similarly to what we do with sample data, we can keep the metadata in-memory in the ingesters and apply similar semantics. I propose to use the tenant ID as a hash key, distribute it to the ingesters (taking into account the replication factor), using a hash map to keep a set of the metadata across all instances for a single tenant, and implement a configurable time-based purge process to deal with metadata churn. Given, we need to ensure fair-use we also propose implementing limits for both the number of metadata entries we can receive and the size of a single entry.\nRead Path In my eyes, the read path seems to only have one option. At the moment of writing, Cortex uses a DummyTargetRetriever as a way to signal that these API endpoints are not implemented. We’d need to modify the Prometheus interface to support a Context and extract the tenant ID from there. Then, use the tenant ID to query the ingesters for the data, deduplicate it and serve it.\nConclusions I conclude that solution #2 is ideal for this work on the write path. It allows us to use similar semantics to samples, thus reducing operational complexity, and lays a groundwork for when we start receiving metadata alongside samples.\nThere’s one last piece to address: Allowing metadata to survive rolling restarts. Option #1 handles this well, given the aim would be to use an external cache such as Memcached. Option #2 lacks this, as it does not include any plans to persist this data. Given Prometheus (by default) sends metadata every minute, and we don’t need a high level of consistency. We expect that an eventual consistency of up to 1 minute on the default case is deemed acceptable.\nReferences Prometheus Propagate metadata via Remote Write Design Doc Prometheus Propagate metadata via Remote Write Design Issue ","categories":"","description":"","excerpt":" Author: @gotjosh Reviewers: @gouthamve, @pracucci Date: March 2020 …","ref":"/docs/proposals/support-metadata-api/","tags":"","title":"Support metadata API"},{"body":" Author: Ilan Gofman Date: June 2021 Status: Proposal Problem Currently, Cortex only implements a time series deletion API for chunk storage. We present a design for implementing time series deletion with block storage. We would like to have the same API for deleting series as currently implemented in Prometheus and in Cortex with chunk storage.\nThis can be very important for users to have as confidential or accidental data might have been incorrectly pushed and needs to be removed. As well as potentially removing high cardinality data that is causing inefficient queries.\nRelated works As previously mentioned, the deletion feature is already implemented with chunk storage. The main functionality is implemented through the purger service. It accepts requests for deletion and processes them. At first, when a deletion request is made, a tombstone is created. This is used to filter out the data for queries. After some time, a deletion plan is executed where the data is permanently removed from chunk storage.\nCan find more info here:\nCortex documentation for chunk store deletion Chunk deletion proposal Background on current storage With a block-storage configuration, Cortex stores data that could be potentially deleted by a user in:\nObject store (GCS, S3, etc..) for long term storage of blocks Ingesters for more recent data that should be eventually transferred to the object store Cache Index cache Metadata cache Chunks cache (stores the potentially to be deleted data) Query results cache (stores the potentially to be deleted data) Compactor during the compaction process Store-gateway Proposal The deletion will not happen right away. Initially, the data will be filtered out from queries using tombstones and will be deleted afterward. This will allow the user some time to cancel the delete request.\nAPI Endpoints The existing purger service will be used to process the incoming requests for deletion. The API will follow the same structure as the chunk storage endpoints for deletion, which is also based on the Prometheus deletion API.\nThis will enable the following endpoints for Cortex when using block storage:\nPOST /api/v1/admin/tsdb/delete_series - Accepts Prometheus style delete request for deleting series.\nParameters:\nstart=\u003crfc3339 | unix_timestamp\u003e Optional. If not provided, will be set to minimum possible time. end=\u003crfc3339 | unix_timestamp\u003e Optional. If not provided, will be set to maximum possible time (time when request was made). End time cannot be greater than the current UTC time. match[]=\u003cseries_selector\u003e Cannot be empty, must contain at least one label matcher argument. POST /api/v1/admin/tsdb/cancel_delete_request - To cancel a request if it has not been processed yet for permanent deletion. This can only be done before the -purger.delete-request-cancel-period has passed. Parameters:\nrequest_id GET /api/v1/admin/tsdb/delete_series - Get all delete requests id’s and their current status.\nPrometheus also implements a clean_tombstones API which is not included in this proposal. The tombstones will be deleted automatically once the permanent deletion has taken place which is described in the section below. By default, this should take approximately 24 hours.\nDeletion Lifecycle The deletion request lifecycle can follow these 3 states:\nPending - Tombstone file is created. During this state, the queriers will be performing query time filtering. The initial time period configured by -purger.delete-request-cancel-period, no data will be deleted. Once this period is over, permanent deletion processing will begin and the request is no longer cancellable. Processed - All requested data has been deleted. Initially, will still need to do query time filtering while waiting for the bucket index and store-gateway to pick up the new blocks. Once that period has passed, will no longer require any query time filtering. Deleted - The deletion request was cancelled. A grace period configured by -purger.delete-request-cancel-period will allow the user some time to cancel the deletion request if it was made by mistake. The request is no longer cancelable after this period has passed. Filtering data during queries while not yet deleted: Once a deletion request is received, a tombstone entry will be created. The object store such as S3, GCS, Azure storage, can be used to store all the deletion requests. See the section below for more detail on how the tombstones will be stored. Using the tombstones, the querier will be able to filter the to-be-deleted data initially. If a cancel delete request is made, then the tombstone file will be deleted. In addition, the existing cache will be invalidated using cache generation numbers, which are described in the later sections.\nThe compactor’s BlocksCleaner service will scan for new tombstone files and will update the bucket-index with the tombstone information regarding the deletion requests. This will enable the querier to periodically check the bucket index if there are any new tombstone files that are required to be used for filtering. One drawback of this approach is the time it could take to start filtering the data. Since the compactor will update the bucket index with the new tombstones every -compactor.cleanup-interval (default 15 min). Then the cached bucket index is refreshed in the querier every -blocks-storage.bucket-store.sync-interval (default 15 min). Potentially could take almost 30 min for queriers to start filtering deleted data when using the default values. If the information requested for deletion is confidential/classified, the time delay is something that the user should be aware of, in addition to the time that the data has already been in Cortex.\nAn additional thing to consider is that this would mean that the bucket-index would have to be enabled for this API to work. Since the plan is to make to the bucket-index mandatory in the future for block storage, this shouldn’t be an issue.\nSimilar to the chunk storage deletion implementation, the initial filtering of the deleted data will be done inside the Querier. This will allow filtering the data read from both the store gateway and the ingester. This functionality already exists for the chunk storage implementation. By implementing it in the querier, this would mean that the ruler will be supported too (ruler internally runs the querier).\nStoring tombstones in object store The Purger will write the new tombstone entries in a separate folder called tombstones in the object store (e.g. S3 bucket) in the respective tenant folder. Each tombstone can have a separate JSON file outlining all the necessary information about the deletion request such as the parameters passed in the request, as well as some meta-data such as the creation date of the file. The name of the file can be a hash of the API parameters (start, end, markers). This way if a user calls the API twice by accident with the same parameters, it will only create one tombstone. To keep track of the request state, filename extensions can be used. This will allow the tombstone files to be immutable. The 3 different file extensions will be pending, processed, deleted. Each time the deletion request moves to a new state, a new file will be added with the same deletion information but a different extension to indicate the new state. The file containing the previous state will be deleted once the new one is created. If a deletion request is cancelled, then a tombstone file with the .deleted filename extension will be created.\nWhen it is determined that the request should move to the next state, then it will first write a new file containing the tombstone information to the object store. The information inside the file will be the same except the stateCreationTime, which is replaced with the current timestamp. The extension of the new file will be different to reflect the new state. If the new file is successfully written, the file with the previous state is deleted. If the write of the new file fails, then the previous file is not going to be deleted. Next time the service runs to check the state of each tombstone, it will retry creating the new file with the updated state. If the write is successful but the deletion of the old file is unsuccessful then there will be 2 tombstone files with the same filename but different extension. When BlocksCleaner writes the tombstones to the bucket index, the compactor will check for duplicate tombstone files but with different extensions. It will use the tombstone with the most recently updated state and try to delete the file with the older state. There could be a scenario where there are two files with the same request ID but different extensions: {.pending, .processed} or {.pending, .deleted}. In this case, the .processed or .deleted file will be selected as it is always the later state compared to the pending state.\nThe tombstone will be stored in a single JSON file per request and state:\n/\u003ctenantId\u003e/tombstones/\u003crequest_id\u003e.json.\u003cstate\u003e The schema of the JSON file is:\n{ \"requestId\": \u003cstring\u003e, \"startTime\": \u003cint\u003e, \"endTime\": \u003cint\u003e, \"requestCreationTime\": \u003cint\u003e, \"stateCreationTime\": \u003cint\u003e, \"matchers\": [ \"\u003cstring matcher 1\u003e\", .., \"\u003cstring matcher n\u003e\" ] }, \"userID\": \u003cstring\u003e, } Pros:\nAllows deletion and un-delete to be done in a single operation. Cons:\nNegative impact on query performance when there are active tombstones. As in the chunk storage implementation, all the series will have to be compared to the matchers contained in the active tombstone files. The impact on performance should be the same as the deletion would have with chunk storage.\nWith the default config, potential 30 minute wait for the data to begin filtering if using the default configuration.\nInvalidating cache Using block store, the different caches available are:\nIndex cache Metadata cache Chunks cache (stores the potentially to be deleted chunks of data) Query results cache (stores the potentially to be deleted data) There are two potential caches that could contain deleted data, the chunks cache, and the query results cache. Using the tombstones, the queriers filter out the data received from the ingesters and store-gateway. The cache not being processed through the querier needs to be invalidated to prevent deleted data from coming up in queries.\nFirstly, the query results cache needs to be invalidated for each new delete request or a cancellation of one. This can be accomplished by utilizing cache generation numbers. For each tenant, their cache is prefixed with a cache generation number. When the query front-end discovers a cache generation number that is greater than the previous generation number, then it knows to invalidate the query results cache. However, the cache can only be invalidated once the queriers have loaded the tombstones from the bucket index and have begun filtering the data. Otherwise, to-be deleted data might show up in queries and be cached again. One of the way to guarantee that all the queriers are using the new tombstones is to wait until the bucket index staleness period has passed from the time the tombstones have been written to the bucket index. The staleness period can be configured using the following flag: -blocks-storage.bucket-store.bucket-index.max-stale-period. We can use the bucket index staleness period as the delay to wait before the cache generation number is increased. A query will fail inside the querier, if the bucket index last update is older the staleness period. Once this period is over, all the queriers should have the updated tombstones and the query results cache can be invalidated. Here is the proposed method for accomplishing this:\nThe cache generation number will be a timestamp. The default value will be 0. The bucket index will store the cache generation number. The query front-end will periodically fetch the bucket index. Inside the compactor, the BlocksCleaner will load the tombstones from object store and update the bucket index accordingly. It will calculate the cache generation number by iterating through all the tombstones and their respective times (next bullet point) and selecting the maximum timestamp that is less than (current time minus -blocks-storage.bucket-store.bucket-index.max-stale-period). This would mean that if a deletion request is made or cancelled, the compactor will only update the cache generation number once the staleness period is over, ensuring that all queriers have the updated tombstones. For requests in a pending or processed state, the requestCreationTime will be used when comparing the maximum timestamps. If a request is in a deleted state, it will use the stateCreationTime for comparing the timestamps. This means that the cache gets invalidated only once it has been created or deleted, and the bucket index staleness period has passed. The cache will not be invalidated again when a request advances from pending to processed state. The query front-end will fetch the cache generation number from the bucket index. The query front end will compare it to the current cache generation number stored in the front-end. If the cache generation number from the front-end is less than the one from bucket index, then the cache is invalidated. In regards to the chunks cache, since it is retrieved from the store gateway and passed to the querier, it will be filtered out like the rest of the time series data in the querier using the tombstones, with the mechanism described in the previous section.\nPermanently deleting the data The proposed approach is to perform the deletions from the compactor. A new background service inside the compactor called DeletedSeriesCleaner can be created and is responsible for executing the deletion.\nProcessing This will happen after a grace period has passed once the API request has been made. By default this should be 24 hours. A background task can be created to process the permanent deletion of time series. This background task can be executed each hour.\nTo delete the data from the blocks, the same logic as the Bucket Rewrite Tool from Thanos can be leveraged. This tool does the following: tools bucket rewrite rewrites chosen blocks in the bucket, while deleting or modifying series. The tool itself is a CLI tool that we won’t be using, but instead we can utilize the logic inside it. For more information about the way this tool runs, please see the code here.\nThe compactor’s DeletedSeriesCleaner will apply this logic on individual blocks and each time it is run, it creates a new block without the data that matched the deletion request. The original individual blocks containing the data that was requested to be deleted, need to be marked for deletion by the compactor.\nWhile deleting the data permanently from the block storage, the meta.json files will be used to keep track of the deletion progress. Inside each meta.json file, we will add a new field called tombstonesFiltered. This will store an array of deletion request id’s that were used to create this block. Once the rewrite logic is applied to a block, the new block’s meta.json file will append the deletion request id(s) used for the rewrite operation inside this field. This will let the DeletedSeriesCleaner know that this block has already processed the particular deletions requests listed in this field. Assuming that the deletion requests are quite rare, the size of the meta.json files should remain small.\nThe DeletedSeriesCleaner can iterate through all the blocks that the deletion request could apply to. For each of these blocks, if the deletion request ID isn’t inside the meta.json tombstonesFiltered field, then the compactor can apply the rewrite logic to this block. If there are multiple tombstones that are currently being processing for deletions and apply to a particular block, then the DeletedSeriesCleaner will process both at the same time to prevent additional blocks from being created. If after iterating through all the blocks, it doesn’t find any such blocks requiring deletion, then the Pending state is complete and the request progresses to the Processed state.\nOne important thing to note regarding this rewrite tool is that it should not be used at the same time as when another compactor is touching a block. If the tool is run at the same time as compaction on a particular block, it can cause overlap and the data marked for deletion can already be part of the compacted block. To mitigate such issues, these are some of the proposed solutions:\nOption 1: Only apply the deletion once the blocks are in the final state of compaction.\nPros:\nSimpler implementation as everything is contained within the DeletedSeriesCleaner. Cons:\nMight have to wait for a longer period of time for the compaction to be finished. This would mean the earliest time to be able to run the deletion would be once the last time from the block_ranges in the compactor_config has passed. By default this value is 24 hours, so only once 24 hours have passed and the new compacted blocks have been created, then the rewrite can be safely run. Option 2: For blocks that still need to be compacted further after the deletion request cancel period is over, the deletion logic can be applied before the blocks are compacted. This will generate a new block which can then be used instead for compaction with other blocks.\nPros:\nThe deletion can be applied earlier than the previous options. Only applies if the deletion request cancel period is less than the last time interval for compaction is. Cons: Added coupling between the compaction and the DeletedSeriesCleaner. Might block compaction for a short time while doing the deletion. Once all the applicable blocks have been rewritten without the deleted data, the deletion request state moves to the Processed state. Once in this state, the queriers will still have to perform query time filtering using the tombstones until the old blocks that were marked for deletion are no longer queried by the queriers. This will mean that the query time filtering will last for an additional length of -compactor.deletion-delay + -compactor.cleanup-interval + -blocks-storage.bucket-store.sync-interval in the Processed state. Once that time period has passed, the queriers should no longer be querying any of the old blocks that were marked for deletion. The tombstone will no longer be used after this.\nCancelled Delete Requests If a request was successfully cancelled, then a tombstone file a .deleted extension is created. This is done to help ensure that the cache generation number is updated and the query results cache is invalidated. The compactor’s blocks cleaner can take care of cleaning up .deleted tombstones after a period of time of when they are no longer required for cache invalidation. This can be done after 10 times the bucket index max staleness time period has passed. Before removing the file from the object store, the current cache generation number must greater than or equal to when the tombstone was cancelled.\nHandling failed/unfinished delete jobs: Deletions will be completed and the tombstones will be deleted only when the DeletedSeriesCleaner iterates over all blocks that match the time interval and confirms that they have been re-written without the deleted data. Otherwise, it will keep iterating over the blocks and process the blocks that haven’t been rewritten according to the information in the meta.json file. In case of any failure that causes the deletion to stop, any unfinished deletions will be resumed once the service is restarted. If the block rewrite was not completed on a particular block, then the original block will not be marked for deletion. The compactor will continue to iterate over the blocks and process the block again.\nTenant Deletion API If a request is made to delete a tenant, then all the tombstones will be deleted for that user.\nCurrent Open Questions: If the start and end time is very far apart, it might result in a lot of the data being re-written. Since we create a new block without the deleted data and mark the old one for deletion, there may be a period of time with lots of extra blocks and space used for large deletion queries. There will be a delay between the deletion request and the deleted data being filtered during queries. In Prometheus, there is no delay. One way to filter out immediately is to load the tombstones during query time but this will cause a negative performance impact. Adding limits to the API such as: Max number of deletion requests allowed in the last 24 hours for a given tenent. Max number of pending tombstones for a given tenant. Alternatives Considered Adding a Pre-processing State The process of permanently deleting the data can be separated into 2 stages, preprocessing and processing.\nPre-processing will begin after the -purger.delete-request-cancel-period has passed since the API request has been made. The deletion request will move to a new state called BuildingPlan. The compactor will outline all the blocks that may contain data to be deleted. For each separate block that the deletion may be applicable to, the compactor will begin the process by adding a series deletion marker inside the series-deletion-marker.json file. The JSON file will contain an array of deletion request id’s that need to be applied to the block, which allows the ability to handle the situation when there are multiple tombstones that could be applicable to a particular block. Then during the processing step, instead of checking the meta.json file, we only need to check if a marker file exists with a specific deletion request id. If the marker file exists, then we apply the rewrite logic.\nAlternative Permanent Deletion Processing For processing the actual deletions, an alternative approach is not to wait until the final compaction has been completed and filter out the data during compaction. If the data is marked to be deleted, then don’t include it the new bigger block during compaction. For the remaining blocks where the data wasn’t filtered during compaction, the deletion can be done the same as in the previous section.\nPros:\nThe deletion can happen sooner. The rewrite tools creates additional blocks. By filtering the metrics during compaction, the intermediary re-written block will be avoided. Cons:\nA more complicated implementation requiring add more logic to the compactor Slower compaction if it needs to filter all the data Need to manage which blocks should be deleted with the rewrite vs which blocks already had data filtered during compaction. Would need to run the rewrite logic during and outside of compaction because some blocks that might need to be deleted are already in the final compaction state. So that would mean the deletion functionality has to be implemented in multiple places. Won’t be leveraging the rewrites tools from Thanos for all the deletion, so potentially more work is duplicated ","categories":"","description":"","excerpt":" Author: Ilan Gofman Date: June 2021 Status: Proposal Problem …","ref":"/docs/proposals/block-storage-time-series-deletion/","tags":"","title":"Time Series Deletion from Blocks Storage"},{"body":" Author: @roystchiang Reviewers: Date: August 2022 Status: Proposed Timeseries Partitioning in Compactor Introduction The compactor is a crucial component in Cortex responsible for deduplication of replicated data, and merging blocks across multiple time intervals together. This proposal will not go into great depth with why the compactor is necessary, but aims to focus on how to scale the compactor as a tenant grows within a Cortex cluster.\nProblem and Requirements Cortex introduced horizontally scaling compactor which allows multiple compactors to compact blocks for a single tenant, sharded by time interval. The compactor is capable of compacting multiple smaller blocks into a larger block, to reduce the duplicated information in index. The following is an illustration of how the shuffle sharding compactor works, where each arrow represents a single compaction that can be carried out independently. However, if the tenant is sending unique timeseries, the compaction process does not help with reducing the index size. Furthermore, this scaling of parallelism by time interval is not sufficient for a tenant with hundreds of millions of timeseries, as more timeseries means longer compaction time.\nCurrently, the compactor is bounded by the 64GB index size, and having a compaction that takes days to complete simply is not sustainable. This time includes the time to download the blocks, merging of the timeseries, writing to disk, and finally uploading to object storage.\nThe compactor is able to compact up to 400M timeseries within 12 hours, and will fail with the error of index exceeding 64GB. Depending on the number of labels and the size of labels, one might reach the 64GB limit sooner. We need a solution that is capable of:\nhandling the 64GB index limit reducing the overall compaction time downloading the data in smaller batches reducing the time required to compact Design A reminder of what a Prometheus TSDB is composed of: an index and chunks. An index is a mapping of timeseries to the chunks, so we can do a direct lookup in the chunks. Each timeseries is effectively a set of labels, mapped to a list of \u003ctimestamp, value\u003e pair. This proposal focuses on partitioning of the timeseries.\nPartitioning strategy The compactor will compact a overlapping time-range into multiple sub-blocks, instead of a single block. Cortex can determine which partition a single timeseries should go into by applying a hash to the timeseries label, and taking the modulo of the hash by the number of partition. This guarantees that with same number of partition, the same timeseries will go into the same partition.\npartitionId = Hash(timeseries label) % number of partition\nThe number of partition will be determined automatically, via a configured multiplier. This multiplier factor allows us to group just a subset of the blocks together to achieve the same deduplication factor as having all the blocks. Using a multiplier of 2 as an example, we can do grouping for partition of 2, 4 and 8. We’ll build on the actual number of partition determination in a later section.\nDetermining overlapping blocks In order to reduce the amount of time spent downloading blocks, and iterating through the index to filter out unrelated timeseries, we can do smart grouping of the blocks.\nGiven that we are always multiplying the number of partition by the multiplier factor, we can deduce from the modulo which partition could contain overlapping result\nGiven a hash N, if N % 8 == 7, then N % 4 must be 3 Given a hash N, if N % 8 == 3, then N % 4 must be 3 Given a hash N, if N % 8 == 4, then N % 4 must be 0 Given a hash N, if N % 8 == 0, then N % 4 must be 0 Hence it is safe to group blocks with N % 8 == 7 with N % 8 == 3 together with N % 4 == 3 together, and we are sure that other blocks won’t contain the same timeseries. We also know that if N % 8 == 0, then we don’t need to download blocks where N % 4 == 1 or N % 4 == 2 Given partition count and partition id, we can immediately find out which blocks are required. Using the above modulo example, we get the following partitiong mapping.\nPlanning the compaction The shuffle sharding compactor introduced additional logic to group blocks by distinct time intervals. It can also sum up the sizes of all indices to determine how many shards are required in total. Using the above example again, and assuming that each block has an index of 30GB, the sum is 30GB * 14 = 420GB, which needs to be at least 7, since maximum index size is 64GB. Using the multiplier factor, it will be rounded up to 8.\nNow the planner knows the resulting compaction will have 8 partitions, it can start planning out which groups of blocks can go into a single compaction group. Given that we need 8 partitions in total, the planner will go through the process above to find out what blocks are necessary. Using the above example again, but we have distinct time intervals, T1, T2, and T3. T1 has 2 partitions, T2 has 4 partitions, and T3 has 8 partitions, and we want to produce T1-T3 blocks Compaction Group 1-8 T1 - Partition 1-2 T2 - Partition 1-4 T3 - Partition 1-8 Compaction Group 2-8 T1 - Partition 2-2 T2 - Partition 2-4 T3 - Partition 2-8 Compaction Group 3-8 T1 - Partition 1-2 T2 - Parittion 3-4 T3 - Partition 3-8 Compaction Group 4-8 T1 - Partition 2-2 T2 - Partition 4-4 T3 - Partition 4-8 Compaction Group 5-8 T1 - Partition 1-2 T2 - Partition 1-4 T3 - Partition 5-8 Compaction Group 6-8 T1 - Partition 2-2 T2 - Partition 2-4 T3 - Partition 6-8 Compaction Group 7-8 T1 - Partition 1-2 T2 - Partition 3-4 T3 - Partition 7-8 Compaction Group 8-8 T1 - Partition 2-2 T2 - Partition 4-4 T3 - Partition 8-8 T1 - Partition 1-2 is used in multiple compaction groups, and the following section will describe how the compaction avoids duplicate timeseries in the resulting blocks\nCompaction Now that the planner has produced a compaction plan for the T1-T3 compaction groups, the compactor can start downloading the necessary blocks. Using compaction group 1-8 from above as example. T1 - Partition 1-2 was created with hash % 2 == 0, and in order to avoid having duplication information in blocks produced by compaction group 3-8, compaction group 5-8, and compaction group 7-8, we need apply the filter the %8 == 0 hash, as that’s the hash of the highest partition count.\nCompaction Workflow Compactor initializes Grouper and Planner. Compactor retrieves block’s meta.json and call Grouper to group blocks for compaction. Grouper generates partitioned compaction groups: Grouper groups source blocks into unpartitioned groups. For each unpartitioned group: Generates partitioned compaction group ID which is hash of min and max time of result block. If the ID exists under the tenant directory in block storage, continue on next unpartitioned group. Calculates number of partitions. Number of partitions indicates how many partitions one unpartitioned group would be partitioned into based on the total size of indices and number of time series from each source blocks in the unpartitioned group. Assign source blocks into each partition with partition ID (value is in range from 0 to number_of_partitions - 1). Note that one source block could be used in multiple partitions (explanation in Planning the compaction and Compaction). So multiple partition ID could be assigned to same source block. Check more partitioning examples in Compaction Partitioning Examples Generates partitioned compaction group that indicates which partition ID each blocks got assigned. Partitioned compaction group information would be stored in block storage under the tenant directory it belongs to and the stored file can be picked up by cleaner later. Partitioned compaction group information contains partitioned compaction group ID, number of partitions, list of partitions which has partition ID and list of source blocks. Store partitioned compaction group ID in block storage under each blocks’ directory that are used by the generated partitioned compaction group. Grouper returns partitioned compaction groups to Compactor. Each returned group would have partition ID, number of partitions, and list of source blocks in memory. Compactor iterates over each partitioned compaction group. For each iteration, calls Planner to make sure the group is ready for compaction. Planner collects partitioned compaction group which is ready for compaction. For each partitions in the group and for each blocks in the partition: Make sure all source blocks fit within the time range of the group. Make sure each source block with assigned partition IDs is currently not used by another ongoing compaction. This could utilize visit marker file that is introduced in #4805 by expanding it for each partition ID of the source block. If all blocks in the partition are ready to be compacted, mark status of those blocks with assigned partition ID as pending. The status information of each partition ID would be stored in block storage under the corresponding block directory in order for cleaner to pick it up later. If not all blocks in the partition are ready, continue on next partition Return all ready partitions to Compactor. Compactor starts compacting partitioned blocks. Once compaction completed, Compactor would mark status of all blocks along with assigned partition ID in the group as completed. Compactor should use partitioned compaction group ID to retrieve partitioned compaction group information from block storage to get all partition IDs assigned to each block. Then, retrieve status information of each partition ID this assigned to block under current block directory in block storage. If all assigned partition ID of the block have status set to completed, upload deletion marker for this block. Otherwise, no deletion marker would be uploaded. Clean up Workflow Cleaner would periodically check any tenants having deletion marker. If there is a deletion marker for the tenant, Cleaner should remove all blocks and then clean up other files including partitioned group information files after tenant clean up delay. If there is no deletion marker for tenant, Clean should scan any source blocks having a deletion marker. If there is a deletion marker for the block, Cleaner should delete it.\nPerformance Currently a 400M timeseries takes 12 hours to compact, without taking block download into consideration. If we have a partition count of 2, we can reduce this down to 6 hours, and a partition count of 10 is 3 hours. The scaling is not linear, and I’m still attempting to find out why. The initial result is promising enough to continue though.\nAlternatives Considered Dynamic Number of Partition We can also increase/decrease the number of partition without needing the multiplier factor. However, if a tenant is sending highly varying number of timeseries or label size, the index size can be very different, resulting in highly dynamic number of partitions. To perform deduplication, we’ll end up having to download all the sub-blocks, and it can be inefficient as less parallelization can be done, and we will spend more time downloading all the unnecessary blocks.\nConsistent Hashing Jump consistent hash, rendezvous hashing, and other consistent hashing are great algorithms to avoid reshuffling of data when introducing/removing partitions on the fly. However, it does not bring much of a benefit when determining which partition contains the same timeseries, which we need to deduplication of index.\nPartition by metric name It is likely that when a query comes, a tenant is interested in an aggregation of a single metric, across all label names. The compactor can partition by metric name, so that all timeseries with the same name will go into the same block. However, this can result in very uneven partitioning.\nArchitecture Planning The shuffle partitioning compactor introduced a planner logic, which we can extend on. This planner is responsible for grouping the blocks together by time interval, in order to compact blocks in parallel. The grouper can also determine the number of partition by looking at the sum of index file sizes. In addition, it can also do the grouping of the sub-blocks together, so we can achieve even higher parallelism.\nClean up Cortex compactor cleans up obsolete source blocks by looking at a deletion marker. The current architecture does not have the problem of having a single source block involved in multiple compaction. However, this proposal is able to achieve higher parallelism than before, hence it is possible that a source block is involved multiple times. Changes needs to be made on the compactor regarding how many plans a particular blocked is involved in, and determining when a block is safe to be deleted.\nChanges Required in Dependencies Partitioning during compaction time Prometheus exposes the possibility to pass in a custom mergeFunc. This allows us to plug in the custom partitioning strategy. However, the mergeFunc is only called when the timeseries is successfully replicated to at least 3 replicas, meaning that we’ll produce duplicate timeseries across blocks if the data is only replicated once. To work around the issue, we can propose Prometheus to allow the configuration of the MergeChunkSeriesSet.\nSource block checking Cortex uses Thanos’s compactor logic, and it has a check to make sure the source blocks of the input blocks do not overlap. Meaning that if BlockA is produced from BlockY, and BlockB is also produced from BlockY, it will halt. This is not desirable for us, since partitioning by timeseries means the same source blocks will produce multiple blocks. Reason for having this check in Thanos is supposed to prevent having duplicate chunks, but the change was introduced without knowing whether it will actually help. We’ll need to introduce a change in Thanos to disable this check, or start using Thanos compactor as a library instead of a closed box.\nWork Plan Performance test the impact on query of having multiple blocks Get real data on the efficiency of modulo operator for partitioning Get the necessary changes in Prometheus approved and merged Get the necessary changes in Thanos approved and merged Implement the number of partition determination in group Implement the grouper logic in Cortex Implement the clean up logic in Cortex Implement the partitioning strategy in Cortex, passed to Prometheus Produce the partitioned blocks Appendix Risks Is taking the modulo of the hash sufficient to produce a good distribution of partitions? What’s the effect of having too many blocks for the same time range? Frequently Asked Questions Are we able to decrease the number of partition? Using partitions of 2, and 4, and 8 as example T1 partition 1 - Hash(timeseries label) % 2 == 0 T1 partition 2 - Hash(timeseries label) % 2 == 1 T2 partition 1 - Hash(timeseries label) % 4 == 0 T2 partition 2 - Hash(timeseries label) % 4 == 1 T2 partition 3 - Hash(timeseries label) % 4 == 2 T2 partition 4 - Hash(timeseries label) % 4 == 3 T3 partition 1 - Hash(timeseries label) % 8 == 0 T3 partition 2 - Hash(timeseries label) % 8 == 1 T3 partition 3 - Hash(timeseries label) % 8 == 2 T3 partition 4 - Hash(timeseries label) % 8 == 3 T3 partition 5 - Hash(timeseries label) % 8 == 4 T3 partition 6 - Hash(timeseries label) % 8 == 5 T3 partition 7 - Hash(timeseries label) % 8 == 6 T3 partition 8 - Hash(timeseries label) % 8 == 7 We are free to produce a resulting timerange T1-T3, without having to download all 14 blocks in a single compactor If T1-T3 can fit inside 4 partitions, we can do the following grouping T1 partition 1 - Hash(timeseries label) % 2 == 0 \u0026\u0026 % 4 == 0 T2 partition 1 - Hash(timeseries label) % 4 == 0 \u0026\u0026 T3 partition 1 - Hash(timeseries label) % 8 == 0 T3 partition 5 - Hash(timeseries label) % 8 == 4 T1 partition 2 - Hash(timeseries label) % 2 == 1 \u0026\u0026 % 4 == 01 T2 partition 2 - Hash(timeseries label) % 4 == 1 T3 partition 2 - Hash(timeseries label) % 8 == 1 T3 partition 7 - Hash(timeseries label) % 8 == 5 T1 partition 1 - Hash(timeseries label) % 2 == 0 \u0026\u0026 % 4 == 2 T2 partition 3 - Hash(timeseries label) % 4 == 2 T3 partition 3 - Hash(timeseries label) % 8 == 2 T3 partition 7 - Hash(timeseries label) % 8 == 6 T1 partition 2 - Hash(timeseries label) % 2 == 1 \u0026\u0026 % 4 == 3 T2 partition 4 - Hash(timeseries label) % 4 == 3 T3 partition 4 - Hash(timeseries label) % 8 == 3 T3 partition 8 - Hash(timeseries label) % 8 == 7 If T1-T3 can fit inside 16 partitions, we can do the same grouping, and hash on top T1 partition 1 - Hash(timeseries label) % 2 == 0 \u0026\u0026 % 8 == 0 T2 partition 1 - Hash(timeseries label) % 4 == 0 \u0026\u0026 % 8 == 0 T3 partition 1 - Hash(timeseries label) % 8 == 0 T1 partition 2 - Hash(timeseries label) % 2 == 1 \u0026\u0026 % 8 == 1 T2 partition 2 - Hash(timeseries label) % 4 == 1 \u0026\u0026 % 8 == 1 T3 partition 2 - Hash(timeseries label) % 8 == 1 T1 partition 1 - Hash(timeseries label) % 2 == 0 \u0026\u0026 % 8 == 2 T2 partition 3 - Hash(timeseries label) % 4 == 2 \u0026\u0026 % 8 == 2 T3 partition 3 - Hash(timeseries label) % 8 == 2 T1 partition 2 - Hash(timeseries label) % 2 == 1 \u0026\u0026 % 8 == 3 T2 partition 4 - Hash(timeseries label) % 4 == 3 \u0026\u0026 % 8 == 3 T3 partition 4 - Hash(timeseries label) % 8 == 3 T1 partition 1 - Hash(timeseries label) % 2 == 0 \u0026\u0026 % 8 == 4 T2 partition 1 - Hash(timeseries label) % 4 == 0 \u0026\u0026 % 8 == 4 T3 partition 5 - Hash(timeseries label) % 8 == 4 T1 partition 2 - Hash(timeseries label) % 2 == 1 \u0026\u0026 % 8 == 5 T2 partition 2 - Hash(timeseries label) % 4 == 1 \u0026\u0026 % 8 == 5 T3 partition 6 - Hash(timeseries label) % 8 == 5 T1 partition 1 - Hash(timeseries label) % 2 == 0 \u0026\u0026 % 8 == 6 T2 partition 3 - Hash(timeseries label) % 4 == 2 \u0026\u0026 % 8 == 6 T3 partition 7 - Hash(timeseries label) % 8 == 6 T1 partition 2 - Hash(timeseries label) % 2 == 1 \u0026\u0026 % 8 == 7 T2 partition 4 - Hash(timeseries label) % 4 == 3 \u0026\u0026 % 8 == 7 T3 partition 8 - Hash(timeseries label) % 8 == 7 Compaction Partitioning Examples Scenario: All source blocks were compacted by partitioning compaction (Idea case) All source blocks were previously compacted through partitioning compaction. In this case for each time range, the number of blocks belong to same time range would be 2^x if multiplier is set to 2.\nTime ranges: T1, T2, T3 Source blocks: T1: B1, B2 T2: B3, B4, B5, B6 T3: B7, B8, B9, B10, B11, B12, B13, B14 Total indices size of all source blocks: 200G Number of Partitions = (200G / 64G = 3.125) =\u003e round up to next 2^x = 4\nPartitioning:\nFor T1, there are only 2 blocks which is \u003c 4. So B1 (index 0 in the time range) can be grouped with other blocks having N % 4 == 0 or 2. Because 0 % 2 == 0. B2 (index 1 in the time range) can be grouped with other blocks having N % 4 == 1 or 3. Because 1 % 2 == 1. For T2, B3 (index 0 in the time range) can be grouped with other blocks having N % 4 == 0. B4 (index 1 in the time range) can be grouped with other blocks having N % 4 == 1. B5 (index 2 in the time range) can be grouped with other blocks having N % 4 == 2. B6 (index 3 in the time range) can be grouped with other blocks having N % 4 == 3. For T3, B7 (index 0 in the time range) can be grouped with other blocks having N % 4 == 0. B8 (index 1 in the time range) can be grouped with other blocks having N % 4 == 1. B9 (index 2 in the time range) can be grouped with other blocks having N % 4 == 2. B10 (index 3 in the time range) can be grouped with other blocks having N % 4 == 3. B11 (index 4 in the time range) can be grouped with other blocks having N % 4 == 0. B12 (index 5 in the time range) can be grouped with other blocks having N % 4 == 1. B13 (index 6 in the time range) can be grouped with other blocks having N % 4 == 2. B14 (index 7 in the time range) can be grouped with other blocks having N % 4 == 3. Partitions in Partitioned Compaction Group:\nPartition ID: 0 Number of Partitions: 4 Blocks: B1, B3, B7, B11 Partition ID: 1 Number of Partitions: 4 Blocks: B2, B4, B8, B12 Partition ID: 2 Number of Partitions: 4 Blocks: B1, B5, B9, B13 Partition ID: 3 Number of Partitions: 4 Blocks: B2, B6, B10, B14 Scenario: All source blocks are level 1 blocks All source blocks are level 1 blocks. Since number of level 1 blocks in one time range is not guaranteed to be 2^x, all blocks need to be included in each partition.\nTime ranges: T1 Source blocks: T1: B1, B2, B3 Total indices size of all source blocks: 100G Number of Partitions = (100G / 64G = 1.5625) =\u003e round up to next 2^x = 2\nPartitioning: There is only one time range from all source blocks which means it is compacting level 1 blocks. Partitioning needs to include all source blocks in each partition.\nPartitions in Partitioned Compaction Group:\nPartition ID: 0 Number of Partitions: 2 Blocks: B1, B2, B3 Partition ID: 1 Number of Partitions: 2 Blocks: B1, B2, B3 Scenario: All source blocks are with compaction level \u003e 1 and were generated by compactor without partitioning compaction If source block was generated by compactor without partitioning compaction, there should be only one block per time range. Since there is only one block in one time range, that one block would be included in all partitions.\nTime ranges: T1, T2, T3 Source blocks: T1: B1 T2: B2 T3: B3 Total indices size of all source blocks: 100G Number of Partitions = (100G / 64G = 1.5625) =\u003e round up to next 2^x = 2\nPartitioning:\nFor T1, there is only one source block. Include B1 in all partitions. For T2, there is only one source block. Include B2 in all partitions. For T3, there is only one source block. Include B3 in all partitions. Partitions in Partitioned Compaction Group:\nPartition ID: 0 Number of Partitions: 2 Blocks: B1, B2, B3 Partition ID: 1 Number of Partitions: 2 Blocks: B1, B2, B3 Scenario: All source blocks are with compaction level \u003e 1 and some of them were generated by compactor with partitioning compaction Blocks generated by compactor without partitioning compaction would be included in all partitions. Blocks generated with partitioning compaction would be partitioned based on multiplier.\nTime ranges: T1, T2, T3 Source blocks: T1: B1 (unpartitioned) T2: B2, B3 T3: B4, B5, B6, B7 Total indices size of all source blocks: 100G Number of Partitions = (100G / 64G = 1.5625) =\u003e round up to next 2^x = 2\nPartitioning:\nFor T1, there is only one source block. Include B1 in all partitions. For T2, B2 (index 0 in the time range) can be grouped with other blocks having N % 2 == 0. B3 (index 1 in the time range) can be grouped with other blocks having N % 2 == 1. For T3, B4 (index 0 in the time range) can be grouped with other blocks having N % 2 == 0. B5 (index 1 in the time range) can be grouped with other blocks having N % 2 == 1. B6 (index 2 in the time range) can be grouped with other blocks having N % 2 == 0. B7 (index 3 in the time range) can be grouped with other blocks having N % 2 == 1. Partitions in Partitioned Compaction Group:\nPartition ID: 0 Number of Partitions: 2 Blocks: B1, B2, B4, B6 Partition ID: 1 Number of Partitions: 2 Blocks: B1, B3, B5, B7 ","categories":"","description":"","excerpt":" Author: @roystchiang Reviewers: Date: August 2022 Status: Proposed …","ref":"/docs/proposals/timeseries-partitioning-in-compactor/","tags":"","title":"Timeseries Partitioning in Compactor"},{"body":"Cortex consists of multiple horizontally scalable microservices. Each microservice uses the most appropriate technique for horizontal scaling; most are stateless and can handle requests for any users while some (namely the ingesters) are semi-stateful and depend on consistent hashing. This document provides a basic overview of Cortex’s architecture.\nThe following diagram does not include all the Cortex services, but does represent a typical deployment topology.\nThe role of Prometheus Prometheus instances scrape samples from various targets and then push them to Cortex (using Prometheus’ remote write API). That remote write API emits batched Snappy-compressed Protocol Buffer messages inside the body of an HTTP PUT request.\nCortex requires that each HTTP request bear a header specifying a tenant ID for the request. Request authentication and authorization are handled by an external reverse proxy.\nIncoming samples (writes from Prometheus) are handled by the distributor while incoming reads (PromQL queries) are handled by the querier or optionally by the query frontend.\nBlocks storage The blocks storage is based on Prometheus TSDB: it stores each tenant’s time series into their own TSDB which write out their series to a on-disk Block (defaults to 2h block range periods). Each Block is composed by a few files storing the chunks and the block index.\nThe TSDB chunk files contain the samples for multiple series. The series inside the Chunks are then indexed by a per-block index, which indexes metric names and labels to time series in the chunk files.\nThe blocks storage doesn’t require a dedicated storage backend for the index. The only requirement is an object store for the Block files, which can be:\nAmazon S3 Google Cloud Storage Microsoft Azure Storage OpenStack Swift Local Filesystem (single node only) For more information, please check out the Blocks storage documentation.\nServices Cortex has a service-based architecture, in which the overall system is split up into a variety of components that perform a specific task. These components run separately and in parallel. Cortex can alternatively run in a single process mode, where all components are executed within a single process. The single process mode is particularly handy for local testing and development.\nThe Cortex services are:\nDistributor Ingester Querier Compactor Store gateway Alertmanager (optional) Configs API (optional) Overrides exporter (optional) Query frontend (optional) Query scheduler (optional) Ruler (optional) Distributor The distributor service is responsible for handling incoming samples from Prometheus. It’s the first stop in the write path for series samples. Once the distributor receives samples from Prometheus, each sample is validated for correctness and to ensure that it is within the configured tenant limits, falling back to default ones in case limits have not been overridden for the specific tenant. Valid samples are then split into batches and sent to multiple ingesters in parallel.\nThe validation done by the distributor includes:\nThe metric labels name are formally correct The configured max number of labels per metric is respected The configured max length of a label name and value is respected The timestamp is not older/newer than the configured min/max time range Distributors are stateless and can be scaled up and down as needed.\nHigh Availability Tracker The distributor features a High Availability (HA) Tracker. When enabled, the distributor deduplicates incoming samples from redundant Prometheus servers. This allows you to have multiple HA replicas of the same Prometheus servers, writing the same series to Cortex and then deduplicate these series in the Cortex distributor.\nThe HA Tracker deduplicates incoming samples based on a cluster and replica label. The cluster label uniquely identifies the cluster of redundant Prometheus servers for a given tenant, while the replica label uniquely identifies the replica within the Prometheus cluster. Incoming samples are considered duplicated (and thus dropped) if received by any replica which is not the current primary within a cluster.\nThe HA Tracker requires a key-value (KV) store to coordinate which replica is currently elected. The distributor will only accept samples from the current leader. Samples with one or no labels (of the replica and cluster) are accepted by default and never deduplicated.\nThe supported KV stores for the HA tracker are:\nConsul Etcd Note: Memberlist is not supported. Memberlist-based KV store propagates updates using gossip, which is very slow for HA purposes: result is that different distributors may see different Prometheus server as elected HA replica, which is definitely not desirable.\nFor more information, please refer to config for sending HA pairs data to Cortex in the documentation.\nHashing Distributors use consistent hashing, in conjunction with a configurable replication factor, to determine which ingester instance(s) should receive a given series.\nCortex supports two hashing strategies:\nHash the metric name and tenant ID (default) Hash the metric name, labels and tenant ID (enabled with -distributor.shard-by-all-labels=true) The trade-off associated with the latter is that writes are more balanced across ingesters but each query needs to talk to all ingesters since a metric could be spread across multiple ingesters given different label sets.\nThe hash ring A hash ring (stored in a key-value store) is used to achieve consistent hashing for the series sharding and replication across the ingesters. All ingesters register themselves into the hash ring with a set of tokens they own; each token is a random unsigned 32-bit number. Each incoming series is hashed in the distributor and then pushed to the ingester owning the tokens range for the series hash number plus N-1 subsequent ingesters in the ring, where N is the replication factor.\nTo do the hash lookup, distributors find the smallest appropriate token whose value is larger than the hash of the series. When the replication factor is larger than 1, the next subsequent tokens (clockwise in the ring) that belong to different ingesters will also be included in the result.\nThe effect of this hash set up is that each token that an ingester owns is responsible for a range of hashes. If there are three tokens with values 0, 25, and 50, then a hash of 3 would be given to the ingester that owns the token 25; the ingester owning token 25 is responsible for the hash range of 1-25.\nThe supported KV stores for the hash ring are:\nConsul Etcd Gossip memberlist Quorum consistency Since all distributors share access to the same hash ring, write requests can be sent to any distributor and you can setup a stateless load balancer in front of it.\nTo ensure consistent query results, Cortex uses Dynamo-style quorum consistency on reads and writes. This means that the distributor will wait for a positive response of at least one half plus one of the ingesters to send the sample to before successfully responding to the Prometheus write request.\nLoad balancing across distributors We recommend randomly load balancing write requests across distributor instances. For example, if you’re running Cortex in a Kubernetes cluster, you could run the distributors as a Kubernetes Service.\nIngester The ingester service is responsible for writing incoming series to a long-term storage backend on the write path and returning in-memory series samples for queries on the read path.\nIncoming series are not immediately written to the storage but kept in memory and periodically flushed to the storage (by default, 2 hours). For this reason, the queriers may need to fetch samples both from ingesters and long-term storage while executing a query on the read path.\nIngesters contain a lifecycler which manages the lifecycle of an ingester and stores the ingester state in the hash ring. Each ingester could be in one of the following states:\nPENDING\nThe ingester has just started. While in this state, the ingester doesn’t receive neither write and read requests. JOINING\nThe ingester is starting up and joining the ring. While in this state the ingester doesn’t receive neither write and read requests. The ingester will join the ring using tokens loaded from disk (if -ingester.tokens-file-path is configured) or generate a set of new random ones. Finally, the ingester optionally observes the ring for tokens conflicts and then, once any conflict is resolved, will move to ACTIVE state. ACTIVE\nThe ingester is up and running. While in this state the ingester can receive both write and read requests. LEAVING\nThe ingester is shutting down and leaving the ring. While in this state the ingester doesn’t receive write requests, while it could receive read requests. UNHEALTHY\nThe ingester has failed to heartbeat to the ring’s KV Store. While in this state, distributors skip the ingester while building the replication set for incoming series and the ingester does not receive write or read requests. Ingesters are semi-stateful.\nIngesters failure and data loss If an ingester process crashes or exits abruptly, all the in-memory series that have not yet been flushed to the long-term storage will be lost. There are two main ways to mitigate this failure mode:\nReplication Write-ahead log (WAL) The replication is used to hold multiple (typically 3) replicas of each time series in the ingesters. If the Cortex cluster loses an ingester, the in-memory series held by the lost ingester are also replicated to at least another ingester. In the event of a single ingester failure, no time series samples will be lost. However, in the event of multiple ingester failures, time series may be potentially lost if the failures affect all the ingesters holding the replicas of a specific time series.\nThe write-ahead log (WAL) is used to write to a persistent disk all incoming series samples until they’re flushed to the long-term storage. In the event of an ingester failure, a subsequent process restart will replay the WAL and recover the in-memory series samples.\nContrary to the sole replication and given the persistent disk data is not lost, in the event of multiple ingesters failure each ingester will recover the in-memory series samples from WAL upon subsequent restart. The replication is still recommended in order to ensure no temporary failures on the read path in the event of a single ingester failure.\nIngesters write de-amplification Ingesters store recently received samples in-memory in order to perform write de-amplification. If the ingesters would immediately write received samples to the long-term storage, the system would be very difficult to scale due to the very high pressure on the storage. For this reason, the ingesters batch and compress samples in-memory and periodically flush them out to the storage.\nWrite de-amplification is the main source of Cortex’s low total cost of ownership (TCO).\nQuerier The querier service handles queries using the PromQL query language.\nQueriers fetch series samples both from the ingesters and long-term storage. The ingesters hold the in-memory series that have not yet been flushed to the long-term storage. Because of the replication factor, it is possible that the querier may receive duplicated samples. To address this, the querier internally deduplicates samples with the exact same timestamp for a given time series.\nQueriers are stateless and can be scaled up and down as needed.\nCompactor The compactor is a service which is responsible to:\nCompact multiple blocks of a given tenant into a single optimized larger block. This helps to reduce storage costs (deduplication, index size reduction), and increase query speed (querying fewer blocks is faster). Keep the per-tenant bucket index updated. The bucket index is used by queriers, store-gateways and rulers to discover new blocks in the storage. For more information, see the compactor documentation.\nThe compactor is stateless.\nStore gateway The store gateway is the Cortex service responsible for querying series from blocks. It needs to have an almost up-to-date view of the storage bucket. To discover blocks belonging to their shard, the store gateway can keep the bucket view updated in two different ways:\nPeriodically scanning the bucket (default) Periodically downloading the bucket index For more information, see the store gateway documentation.\nThe store gateway is semi-stateful.\nQuery frontend The query frontend is an optional service providing the querier’s API endpoints and can be used to accelerate the read path. When the query frontend is in place, incoming query requests should be directed to the query frontend instead of the queriers. The querier service will be still required within the cluster, in order to execute the actual queries.\nThe query frontend internally performs some query adjustments and holds queries in an internal queue. In this setup, queriers act as workers which pull jobs from the queue, execute them, and return them to the query-frontend for aggregation. Queriers need to be configured with the query frontend address (via the -querier.frontend-address CLI flag) in order to allow them to connect to the query frontends.\nQuery frontends are stateless. However, due to how the internal queue works, it’s recommended to run a few query frontend replicas to reap the benefit of fair scheduling. Two replicas should suffice in most cases.\nFlow of the query in the system when using query-frontend:\nQuery is received by query frontend, which can optionally split it or serve from the cache. Query frontend stores the query into in-memory queue, where it waits for some querier to pick it up. Querier picks up the query, and executes it. Querier sends result back to query-frontend, which then forwards it to the client. Query frontend can also be used with any Prometheus-API compatible service. In this mode Cortex can be used as an query accelerator with it’s caching and splitting features on other prometheus query engines like Thanos Querier or your own Prometheus server. Query frontend needs to be configured with downstream url address(via the -frontend.downstream-url CLI flag), which is the endpoint of the prometheus server intended to be connected with Cortex.\nQueueing The query frontend queuing mechanism is used to:\nEnsure that large queries, that could cause an out-of-memory (OOM) error in the querier, will be retried on failure. This allows administrators to under-provision memory for queries, or optimistically run more small queries in parallel, which helps to reduce the total cost of ownership (TCO). Prevent multiple large requests from being convoyed on a single querier by distributing them across all queriers using a first-in/first-out queue (FIFO). Prevent a single tenant from denial-of-service-ing (DOSing) other tenants by fairly scheduling queries between tenants. Splitting The query frontend splits multi-day queries into multiple single-day queries, executing these queries in parallel on downstream queriers and stitching the results back together again. This prevents large (multi-day) queries from causing out of memory issues in a single querier and helps to execute them faster.\nCaching The query frontend supports caching query results and reuses them on subsequent queries. If the cached results are incomplete, the query frontend calculates the required subqueries and executes them in parallel on downstream queriers. The query frontend can optionally align queries with their step parameter to improve the cacheability of the query results. The result cache is compatible with any cortex caching backend (currently memcached, redis, and an in-memory cache).\nQuery Scheduler Query Scheduler is an optional service that moves the internal queue from query frontend into separate component. This enables independent scaling of query frontends and number of queues (query scheduler).\nIn order to use query scheduler, both query frontend and queriers must be configured with query scheduler address (using -frontend.scheduler-address and -querier.scheduler-address options respectively).\nFlow of the query in the system changes when using query scheduler:\nQuery is received by query frontend, which can optionally split it or serve from the cache. Query frontend forwards the query to random query scheduler process. Query scheduler stores the query into in-memory queue, where it waits for some querier to pick it up. Querier picks up the query, and executes it. Querier sends result back to query-frontend, which then forwards it to the client. Query schedulers are stateless. It is recommended to run two replicas to make sure queries can still be serviced while one replica is restarting.\nRuler The ruler is an optional service executing PromQL queries for recording rules and alerts. The ruler requires a database storing the recording rules and alerts for each tenant.\nRuler is semi-stateful and can be scaled horizontally. Running rules internally have state, as well as the ring the rulers initiate. However, if the rulers all fail and restart, Prometheus alert rules have a feature where an alert is restored and returned to a firing state if it would have been active in its for period. However, there would be gaps in the series generated by the recording rules.\nAlertmanager The alertmanager is an optional service responsible for accepting alert notifications from the ruler, deduplicating and grouping them, and routing them to the correct notification channel, such as email, PagerDuty or OpsGenie.\nThe Cortex alertmanager is built on top of the Prometheus Alertmanager, adding multi-tenancy support. Like the ruler, the alertmanager requires a database storing the per-tenant configuration.\nAlertmanager is semi-stateful. The Alertmanager persists information about silences and active alerts to its disk. If all of the alertmanager nodes failed simultaneously there would be a loss of data.\nConfigs API The configs API is an optional service managing the configuration of Rulers and Alertmanagers. It provides APIs to get/set/update the ruler and alertmanager configurations and store them into backend. Current supported backend are PostgreSQL and in-memory.\nConfigs API is stateless.\n","categories":"","description":"","excerpt":"Cortex consists of multiple horizontally scalable microservices. Each …","ref":"/docs/architecture/","tags":"","title":"Cortex Architecture"},{"body":"General Notes Cortex has evolved over several years, and the command-line options sometimes reflect this heritage. In some cases the default value for options is not the recommended value, and in some cases names do not reflect the true meaning. We do intend to clean this up, but it requires a lot of care to avoid breaking existing installations. In the meantime we regret the inconvenience.\nDuration arguments should be specified with a unit like 5s or 3h. Valid time units are “ms”, “s”, “m”, “h”.\nQuerier -querier.max-concurrent\nThe maximum number of top-level PromQL queries that will execute at the same time, per querier process. If using the query frontend, this should be set to at least (-querier.worker-parallelism * number of query frontend replicas). Otherwise queries may queue in the queriers and not the frontend, which will affect QoS. Alternatively, consider using -querier.worker-match-max-concurrent to force worker parallelism to match -querier.max-concurrent.\n-querier.timeout\nThe timeout for a top-level PromQL query.\n-querier.max-samples\nMaximum number of samples a single query can load into memory, to avoid blowing up on enormous queries.\nThe next three options only apply when the querier is used together with the Query Frontend or Query Scheduler:\n-querier.frontend-address\nAddress of query frontend service, used by workers to find the frontend which will give them queries to execute.\n-querier.scheduler-address\nAddress of query scheduler service, used by workers to find the scheduler which will give them queries to execute. If set, -querier.frontend-address is ignored, and querier will use query scheduler.\n-querier.dns-lookup-period\nHow often the workers will query DNS to re-check where the query frontend or query scheduler is.\n-querier.worker-parallelism\nNumber of simultaneous queries to process, per query frontend or scheduler. See note on -querier.max-concurrent\n-querier.worker-match-max-concurrent\nForce worker concurrency to match the -querier.max-concurrent option. Overrides -querier.worker-parallelism. See note on -querier.max-concurrent\nQuerier and Ruler -promql.lookback-delta\nTime since the last sample after which a time series is considered stale and ignored by expression evaluations.\nQuery Frontend -querier.align-querier-with-step\nIf set to true, will cause the query frontend to mutate incoming queries and align their start and end parameters to the step parameter of the query. This improves the cacheability of the query results.\n-querier.split-queries-by-day\nIf set to true, will cause the query frontend to split multi-day queries into multiple single-day queries and execute them in parallel.\n-querier.cache-results\nIf set to true, will cause the querier to cache query results. The cache will be used to answer future, overlapping queries. The query frontend calculates extra queries required to fill gaps in the cache.\n-frontend.forward-headers-list\nRequest headers forwarded by query frontend to downstream queriers. Multiple headers may be specified. Defaults to empty.\n-frontend.max-cache-freshness\nWhen caching query results, it is desirable to prevent the caching of very recent results that might still be in flux. Use this parameter to configure the age of results that should be excluded.\n-frontend.memcached.{hostname, service, timeout}\nUse these flags to specify the location and timeout of the memcached cluster used to cache query results.\n-frontend.redis.{endpoint, timeout}\nUse these flags to specify the location and timeout of the Redis service used to cache query results.\nDistributor -distributor.shard-by-all-labels\nIn the original Cortex design, samples were sharded amongst distributors by the combination of (userid, metric name). Sharding by metric name was designed to reduce the number of ingesters you need to hit on the read path; the downside was that you could hotspot the write path.\nIn hindsight, this seems like the wrong choice: we do many orders of magnitude more writes than reads, and ingester reads are in-memory and cheap. It seems the right thing to do is to use all the labels to shard, improving load balancing and support for very high cardinality metrics.\nSet this flag to true for the new behaviour.\nImportant to note is that when setting this flag to true, it has to be set on both the distributor and the querier (called -distributor.shard-by-all-labels on Querier as well). If the flag is only set on the distributor and not on the querier, you will get incomplete query results because not all ingesters are queried.\nUpgrade notes: As this flag also makes all queries always read from all ingesters, the upgrade path is pretty trivial; just enable the flag. When you do enable it, you’ll see a spike in the number of active series as the writes are “reshuffled” amongst the ingesters, but over the next stale period all the old series will be flushed, and you should end up with much better load balancing. With this flag enabled in the queriers, reads will always catch all the data from all ingesters.\nWarning: disabling this flag can lead to a much less balanced distribution of load among the ingesters.\n-distributor.extra-query-delay This is used by a component with an embedded distributor (Querier and Ruler) to control how long to wait until sending more than the minimum amount of queries needed for a successful response.\ndistributor.ha-tracker.enable-for-all-users Flag to enable, for all users, handling of samples with external labels identifying replicas in an HA Prometheus setup. This defaults to false, and is technically defined in the Distributor limits.\ndistributor.ha-tracker.enable Enable the distributors HA tracker so that it can accept samples from Prometheus HA replicas gracefully (requires labels). Global (for distributors), this ensures that the necessary internal data structures for the HA handling are created. The option enable-for-all-users is still needed to enable ingestion of HA samples for all users.\ndistributor.drop-label This flag can be used to specify label names that to drop during sample ingestion within the distributor and can be repeated in order to drop multiple labels.\nRing/HA Tracker Store The KVStore client is used by both the Ring and HA Tracker (HA Tracker doesn’t support memberlist as KV store).\n{ring,distributor.ha-tracker}.prefix The prefix for the keys in the store. Should end with a /. For example with a prefix of foo/, the key bar would be stored under foo/bar. {ring,distributor.ha-tracker}.store Backend storage to use for the HA Tracker (consul, etcd, inmemory, multi). {ring,distributor.ring}.store Backend storage to use for the Ring (consul, etcd, inmemory, memberlist, multi). Consul By default these flags are used to configure Consul used for the ring. To configure Consul for the HA tracker, prefix these flags with distributor.ha-tracker.\nconsul.hostname Hostname and port of Consul. consul.acl-token ACL token used to interact with Consul. consul.client-timeout HTTP timeout when talking to Consul. consul.consistent-reads Enable consistent reads to Consul. etcd By default these flags are used to configure etcd used for the ring. To configure etcd for the HA tracker, prefix these flags with distributor.ha-tracker.\netcd.endpoints The etcd endpoints to connect to. etcd.dial-timeout The timeout for the etcd connection. etcd.max-retries The maximum number of retries to do for failed ops. etcd.tls-enabled Enable TLS. etcd.tls-cert-path The TLS certificate file path. etcd.tls-key-path The TLS private key file path. etcd.tls-ca-path The trusted CA file path. etcd.tls-insecure-skip-verify Skip validating server certificate. etcd.ping-without-stream-allowd' Enable/Disable PermitWithoutStream parameter memberlist Warning: memberlist KV works only for the hash ring, not for the HA Tracker, because propagation of changes is too slow for HA Tracker purposes.\nWhen using memberlist-based KV store, each node maintains its own copy of the hash ring. Updates generated locally, and received from other nodes are merged together to form the current state of the ring on the node. Updates are also propagated to other nodes. All nodes run the following two loops:\nEvery “gossip interval”, pick random “gossip nodes” number of nodes, and send recent ring updates to them. Every “push/pull sync interval”, choose random single node, and exchange full ring information with it (push/pull sync). After this operation, rings on both nodes are the same. When a node receives a ring update, node will merge it into its own ring state, and if that resulted in a change, node will add that update to the list of gossiped updates. Such update will be gossiped R * log(N+1) times by this node (R = retransmit multiplication factor, N = number of gossiping nodes in the cluster).\nIf you find the propagation to be too slow, there are some tuning possibilities (default values are memberlist settings for LAN networks):\nDecrease gossip interval (default: 200ms) Increase gossip nodes (default 3) Decrease push/pull sync interval (default 30s) Increase retransmit multiplication factor (default 4) To find propagation delay, you can use cortex_ring_oldest_member_timestamp{state=\"ACTIVE\"} metric.\nFlags for configuring KV store based on memberlist library:\nmemberlist.nodename Name of the node in memberlist cluster. Defaults to hostname. memberlist.randomize-node-name This flag adds extra random suffix to the node name used by memberlist. Defaults to true. Using random suffix helps to prevent issues when running multiple memberlist nodes on the same machine, or when node names are reused (eg. in stateful sets). memberlist.retransmit-factor Multiplication factor used when sending out messages (factor * log(N+1)). If not set, default value is used. memberlist.join Other cluster members to join. Can be specified multiple times. memberlist.min-join-backoff, memberlist.max-join-backoff, memberlist.max-join-retries These flags control backoff settings when joining the cluster. memberlist.abort-if-join-fails If this node fails to join memberlist cluster, abort. memberlist.rejoin-interval How often to try to rejoin the memberlist cluster. Defaults to 0, no rejoining. Occasional rejoin may be useful in some configurations, and is otherwise harmless. memberlist.left-ingesters-timeout How long to keep LEFT ingesters in the ring. Note: this is only used for gossiping, LEFT ingesters are otherwise invisible. memberlist.leave-timeout Timeout for leaving memberlist cluster. memberlist.gossip-interval How often to gossip with other cluster members. Uses memberlist LAN defaults if 0. memberlist.gossip-nodes How many nodes to gossip with in each gossip interval. Uses memberlist LAN defaults if 0. memberlist.pullpush-interval How often to use pull/push sync. Uses memberlist LAN defaults if 0. memberlist.bind-addr IP address to listen on for gossip messages. Multiple addresses may be specified. Defaults to 0.0.0.0. memberlist.bind-port Port to listen on for gossip messages. Defaults to 7946. memberlist.packet-dial-timeout Timeout used when connecting to other nodes to send packet. memberlist.packet-write-timeout Timeout for writing ‘packet’ data. memberlist.transport-debug Log debug transport messages. Note: global log.level must be at debug level as well. memberlist.gossip-to-dead-nodes-time How long to keep gossiping to the nodes that seem to be dead. After this time, dead node is removed from list of nodes. If “dead” node appears again, it will simply join the cluster again, if its name is not reused by other node in the meantime. If the name has been reused, such a reanimated node will be ignored by other members. memberlist.dead-node-reclaim-time How soon can dead’s node name be reused by a new node (using different IP). Disabled by default, name reclaim is not allowed until gossip-to-dead-nodes-time expires. This can be useful to set to low numbers when reusing node names, eg. in stateful sets. If memberlist library detects that new node is trying to reuse the name of previous node, it will log message like this: Conflicting address for ingester-6. Mine: 10.44.12.251:7946 Theirs: 10.44.12.54:7946 Old state: 2. Node states are: “alive” = 0, “suspect” = 1 (doesn’t respond, will be marked as dead if it doesn’t respond), “dead” = 2. Multi KV This is a special key-value implementation that uses two different KV stores (eg. consul, etcd or memberlist). One of them is always marked as primary, and all reads and writes go to primary store. Other one, secondary, is only used for writes. The idea is that operator can use multi KV store to migrate from primary to secondary store in runtime.\nFor example, migration from Consul to Etcd would look like this:\nSet ring.store to use multi store. Set -multi.primary=consul and -multi.secondary=etcd. All consul and etcd settings must still be specified. Start all Cortex microservices. They will still use Consul as primary KV, but they will also write share ring via etcd. Operator can now use “runtime config” mechanism to switch primary store to etcd. After all Cortex microservices have picked up new primary store, and everything looks correct, operator can now shut down Consul, and modify Cortex configuration to use -ring.store=etcd only. At this point, Consul can be shut down. Multi KV has following parameters:\nmulti.primary - name of primary KV store. Same values as in ring.store are supported, except multi. multi.secondary - name of secondary KV store. multi.mirror-enabled - enable mirroring of values to secondary store, defaults to true multi.mirror-timeout - wait max this time to write to secondary store to finish. Default to 2 seconds. Errors writing to secondary store are not reported to caller, but are logged and also reported via cortex_multikv_mirror_write_errors_total metric. Multi KV also reacts on changes done via runtime configuration. It uses this section:\nmulti_kv_config: mirror_enabled: false primary: memberlist Note that runtime configuration values take precedence over command line options.\nHA Tracker HA tracking has two of its own flags:\ndistributor.ha-tracker.cluster Prometheus label to look for in samples to identify a Prometheus HA cluster. (default “cluster”) distributor.ha-tracker.replica Prometheus label to look for in samples to identify a Prometheus HA replica. (default “__replica__”) It’s reasonable to assume people probably already have a cluster label, or something similar. If not, they should add one along with __replica__ via external labels in their Prometheus config. If you stick to these default values your Prometheus config could look like this (POD_NAME is an environment variable which must be set by you):\nglobal: external_labels: cluster: clustername __replica__: $POD_NAME HA Tracking looks for the two labels (which can be overwritten per user)\nIt also talks to a KVStore and has it’s own copies of the same flags used by the Distributor to connect to for the ring.\ndistributor.ha-tracker.failover-timeout If we don’t receive any samples from the accepted replica for a cluster in this amount of time we will failover to the next replica we receive a sample from. This value must be greater than the update timeout (default 30s) distributor.ha-tracker.store Backend storage to use for the ring (consul, etcd, inmemory, multi). Inmemory only works if there is a single distributor and ingester running in the same process (for testing purposes). (default “consul”) distributor.ha-tracker.update-timeout Update the timestamp in the KV store for a given cluster/replica only after this amount of time has passed since the current stored timestamp. (default 15s) Ingester -ingester.join-after\nHow long to wait in PENDING state during the hand-over process (supported only by the chunks storage). (default 0s)\n-ingester-client.expected-timeseries\nWhen push requests arrive, pre-allocate this many slots to decode them. Tune this setting to reduce memory allocations and garbage. This should match the max_samples_per_send in your queue_config for Prometheus.\n-ingester-client.expected-samples-per-series\nWhen push requests arrive, pre-allocate this many slots to decode them. Tune this setting to reduce memory allocations and garbage. Under normal conditions, Prometheus scrapes should arrive with one sample per series.\n-ingester-client.expected-labels\nWhen push requests arrive, pre-allocate this many slots to decode them. Tune this setting to reduce memory allocations and garbage. The optimum value will depend on how many labels are sent with your timeseries samples.\nRuntime Configuration file Cortex has a concept of “runtime config” file, which is simply a file that is reloaded while Cortex is running. It is used by some Cortex components to allow operator to change some aspects of Cortex configuration without restarting it. File is specified by using -runtime-config.file=\u003cfilename\u003e flag and reload period (which defaults to 10 seconds) can be changed by -runtime-config.reload-period=\u003cduration\u003e flag. Previously this mechanism was only used by limits overrides, and flags were called -limits.per-user-override-config=\u003cfilename\u003e and -limits.per-user-override-period=10s respectively. These are still used, if -runtime-config.file=\u003cfilename\u003e is not specified.\nAt the moment runtime configuration may contain per-user limits, multi KV store, and ingester instance limits.\nExample runtime configuration file:\noverrides: tenant1: ingestion_rate: 10000 max_series_per_metric: 100000 max_series_per_query: 100000 tenant2: max_samples_per_query: 1000000 max_series_per_metric: 100000 max_series_per_query: 100000 multi_kv_config: mirror_enabled: false primary: memberlist ingester_limits: max_ingestion_rate: 42000 max_inflight_push_requests: 10000 When running Cortex on Kubernetes, store this file in a config map and mount it in each services’ containers. When changing the values there is no need to restart the services, unless otherwise specified.\nThe /runtime_config endpoint returns the whole runtime configuration, including the overrides. In case you want to get only the non-default values of the configuration you can pass the mode parameter with the diff value.\nIngester, Distributor \u0026 Querier limits. Cortex implements various limits on the requests it can process, in order to prevent a single tenant overwhelming the cluster. There are various default global limits which apply to all tenants which can be set on the command line. These limits can also be overridden on a per-tenant basis by using overrides field of runtime configuration file.\nThe overrides field is a map of tenant ID (same values as passed in the X-Scope-OrgID header) to the various limits. An example could look like:\noverrides: tenant1: ingestion_rate: 10000 max_series_per_metric: 100000 max_series_per_query: 100000 tenant2: max_samples_per_query: 1000000 max_series_per_metric: 100000 max_series_per_query: 100000 Valid per-tenant limits are (with their corresponding flags for default values):\ningestion_rate_strategy / -distributor.ingestion-rate-limit-strategy\ningestion_rate / -distributor.ingestion-rate-limit\ningestion_burst_size / -distributor.ingestion-burst-size\nThe per-tenant rate limit (and burst size), in samples per second. It supports two strategies: local (default) and global.\nThe local strategy enforces the limit on a per distributor basis, actual effective rate limit will be N times higher, where N is the number of distributor replicas.\nThe global strategy enforces the limit globally, configuring a per-distributor local rate limiter as ingestion_rate / N, where N is the number of distributor replicas (it’s automatically adjusted if the number of replicas change). The ingestion_burst_size refers to the per-distributor local rate limiter (even in the case of the global strategy) and should be set at least to the maximum number of samples expected in a single push request. For this reason, the global strategy requires that push requests are evenly distributed across the pool of distributors; if you use a load balancer in front of the distributors you should be already covered, while if you have a custom setup (ie. an authentication gateway in front) make sure traffic is evenly balanced across distributors.\nThe global strategy requires the distributors to form their own ring, which is used to keep track of the current number of healthy distributor replicas. The ring is configured by distributor: { ring: {}} / -distributor.ring.*.\nmax_label_name_length / -validation.max-length-label-name\nmax_label_value_length / -validation.max-length-label-value\nmax_label_names_per_series / -validation.max-label-names-per-series\nAlso enforced by the distributor, limits on the on length of labels and their values, and the total number of labels allowed per series.\nreject_old_samples / -validation.reject-old-samples\nreject_old_samples_max_age / -validation.reject-old-samples.max-age\ncreation_grace_period / -validation.create-grace-period\nAlso enforce by the distributor, limits on how far in the past (and future) timestamps that we accept can be.\nmax_series_per_user / -ingester.max-series-per-user\nmax_series_per_metric / -ingester.max-series-per-metric\nEnforced by the ingesters; limits the number of active series a user (or a given metric) can have. When running with -distributor.shard-by-all-labels=false (the default), this limit will enforce the maximum number of series a metric can have ‘globally’, as all series for a single metric will be sent to the same replication set of ingesters. This is not the case when running with -distributor.shard-by-all-labels=true, so the actual limit will be N/RF times higher, where N is number of ingester replicas and RF is configured replication factor.\nmax_global_series_per_user / -ingester.max-global-series-per-user\nmax_global_series_per_metric / -ingester.max-global-series-per-metric\nLike max_series_per_user and max_series_per_metric, but the limit is enforced across the cluster. Each ingester is configured with a local limit based on the replication factor, the -distributor.shard-by-all-labels setting and the current number of healthy ingesters, and is kept updated whenever the number of ingesters change.\nRequires -distributor.replication-factor, -distributor.shard-by-all-labels, -distributor.sharding-strategy and -distributor.zone-awareness-enabled set for the ingesters too.\nmax_series_per_query / -ingester.max-series-per-query\nmax_samples_per_query / -ingester.max-samples-per-query\nLimits on the number of timeseries and samples returns by a single ingester during a query.\nmax_metadata_per_user / -ingester.max-metadata-per-user\nmax_metadata_per_metric / -ingester.max-metadata-per-metric Enforced by the ingesters; limits the number of active metadata a user (or a given metric) can have. When running with -distributor.shard-by-all-labels=false (the default), this limit will enforce the maximum number of metadata a metric can have ‘globally’, as all metadata for a single metric will be sent to the same replication set of ingesters. This is not the case when running with -distributor.shard-by-all-labels=true, so the actual limit will be N/RF times higher, where N is number of ingester replicas and RF is configured replication factor.\nmax_fetched_series_per_query / querier.max-fetched-series-per-query When running Cortex with blocks storage this limit is enforced in the queriers on unique series fetched from ingesters and store-gateways (long-term storage).\nmax_global_metadata_per_user / -ingester.max-global-metadata-per-user\nmax_global_metadata_per_metric / -ingester.max-global-metadata-per-metric\nLike max_metadata_per_user and max_metadata_per_metric, but the limit is enforced across the cluster. Each ingester is configured with a local limit based on the replication factor, the -distributor.shard-by-all-labels setting and the current number of healthy ingesters, and is kept updated whenever the number of ingesters change.\nRequires -distributor.replication-factor, -distributor.shard-by-all-labels, -distributor.sharding-strategy and -distributor.zone-awareness-enabled set for the ingesters too.\nIngester Instance Limits Cortex ingesters support limits that are applied per-instance, meaning they apply to each ingester process. These can be used to ensure individual ingesters are not overwhelmed regardless of any per-user limits. These limits can be set under the ingester.instance_limits block in the global configuration file, with command line flags, or under the ingester_limits field in the runtime configuration file.\nAn example as part of the runtime configuration file:\ningester_limits: max_ingestion_rate: 20000 max_series: 1500000 max_tenants: 1000 max_inflight_push_requests: 30000 Valid ingester instance limits are (with their corresponding flags):\nmax_ingestion_rate \\ --ingester.instance-limits.max-ingestion-rate\nLimit the ingestion rate in samples per second for an ingester. When this limit is reached, new requests will fail with an HTTP 500 error.\nmax_series \\ -ingester.instance-limits.max-series\nLimit the total number of series that an ingester keeps in memory, across all users. When this limit is reached, requests that create new series will fail with an HTTP 500 error.\nmax_tenants \\ -ingester.instance-limits.max-tenants\nLimit the maximum number of users an ingester will accept metrics for. When this limit is reached, requests from new users will fail with an HTTP 500 error.\nmax_inflight_push_requests \\ -ingester.instance-limits.max-inflight-push-requests\nLimit the maximum number of requests being handled by an ingester at once. This setting is critical for preventing ingesters from using an excessive amount of memory during high load or temporary slow downs. When this limit is reached, new requests will fail with an HTTP 500 error.\nStorage s3.force-path-style\nSet this to true to force the request to use path-style addressing (http://s3.amazonaws.com/BUCKET/KEY). By default, the S3 client will use virtual hosted bucket addressing when possible (http://BUCKET.s3.amazonaws.com/KEY).\nDNS Service Discovery Some clients in Cortex support service discovery via DNS to find addresses of backend servers to connect to (ie. caching servers). The clients supporting it are:\nBlocks storage’s memcached cache All caching memcached servers Memberlist KV store Supported discovery modes The DNS service discovery, inspired from Thanos DNS SD, supports different discovery modes. A discovery mode is selected adding a specific prefix to the address. The supported prefixes are:\ndns+\nThe domain name after the prefix is looked up as an A/AAAA query. For example: dns+memcached.local:11211 dnssrv+\nThe domain name after the prefix is looked up as a SRV query, and then each SRV record is resolved as an A/AAAA record. For example: dnssrv+_memcached._tcp.memcached.namespace.svc.cluster.local dnssrvnoa+\nThe domain name after the prefix is looked up as a SRV query, with no A/AAAA lookup made after that. For example: dnssrvnoa+_memcached._tcp.memcached.namespace.svc.cluster.local If no prefix is provided, the provided IP or hostname will be used straightaway without pre-resolving it.\nIf you are using a managed memcached service from Google Cloud, or AWS, use the auto-discovery flag instead of DNS discovery, then use the discovery/configuration endpoint as the domain name without any prefix.\nLogging of IP of reverse proxy If a reverse proxy is used in front of Cortex it might be difficult to troubleshoot errors. The following 3 settings can be used to log the IP address passed along by the reverse proxy in headers like X-Forwarded-For.\n-server.log_source_ips_enabled\nSet this to true to add logging of the IP when a Forwarded, X-Real-IP or X-Forwarded-For header is used. A field called sourceIPs will be added to error logs when data is pushed into Cortex.\n-server.log-source-ips-header\nHeader field storing the source IPs. It is only used if -server.log-source-ips-enabled is true and if -server.log-source-ips-regex is set. If not set the default Forwarded, X-Real-IP or X-Forwarded-For headers are searched.\n-server.log-source-ips-regex\nRegular expression for matching the source IPs. It should contain at least one capturing group the first of which will be returned. Only used if -server.log-source-ips-enabled is true and if -server.log-source-ips-header is set. If not set the default Forwarded, X-Real-IP or X-Forwarded-For headers are searched.\n","categories":"","description":"","excerpt":"General Notes Cortex has evolved over several years, and the …","ref":"/docs/configuration/arguments/","tags":"","title":"Cortex Arguments"},{"body":"Cortex adopts some design patterns and code conventions that we ask you to follow when contributing to the project. These conventions have been adopted based on the experience gained over the time and aim to enforce good coding practices and keep a consistent UX (ie. config).\nGo coding style Cortex follows the Go Code Review Comments styleguide and the Formatting and style section of Peter Bourgon’s Go: Best Practices for Production Environments.\nNo global variables Do not use global variables Prometheus metrics When registering a metric:\nDo not use a global variable for the metric Create and register the metric with promauto.With(reg) In any internal Cortex component, do not register the metric to the default prometheus registerer, but pick the registerer in input (ie. NewComponent(reg prometheus.Registerer)) Testing metrics:\nWhen writing using tests, test exported metrics using testutil.GatherAndCompare() Config file and CLI flags conventions Naming:\nConfig file options should be lowercase with words _ (underscore) separated (ie. memcached_client) CLI flags should be lowercase with words - (dash) separated (ie. memcached-client) When adding a new config option, look if a similar one already exists within the config and keep the same naming (ie. addresses for a list of network endpoints) Documentation:\nA CLI flag mentioned in the documentation or changelog should be always prefixed with a single - (dash) ","categories":"","description":"","excerpt":"Cortex adopts some design patterns and code conventions that we ask …","ref":"/docs/contributing/design-patterns-and-code-conventions/","tags":"","title":"Design patterns and Code conventions"},{"body":" The querier service handles queries using the PromQL query language. This document dives into the storage-specific details of the querier service. The general architecture documentation applies too.\nThe querier is stateless.\nHow it works The querier needs to have an almost up-to-date view over the entire storage bucket, in order to find the right blocks to lookup at query time. The querier can keep the bucket view updated in to two different ways:\nPeriodically scanning the bucket (default) Periodically downloading the bucket index Bucket index disabled (default) At startup, queriers iterate over the entire storage bucket to discover all tenants blocks and download the meta.json for each block. During this initial bucket scanning phase, a querier is not ready to handle incoming queries yet and its /ready readiness probe endpoint will fail.\nWhile running, queriers periodically iterate over the storage bucket to discover new tenants and recently uploaded blocks. Queriers do not download any content from blocks except a small meta.json file containing the block’s metadata (including the minimum and maximum timestamp of samples within the block).\nQueriers use the metadata to compute the list of blocks that need to be queried at query time and fetch matching series from the store-gateway instances holding the required blocks.\nBucket index enabled When bucket index is enabled, queriers lazily download the bucket index upon the first query received for a given tenant, cache it in memory and periodically keep it update. The bucket index contains the list of blocks and block deletion marks of a tenant, which is later used during the query execution to find the set of blocks that need to be queried for the given query.\nGiven the bucket index removes the need to scan the bucket, it brings few benefits:\nThe querier is expected to be ready shortly after startup. Lower volume of API calls to object storage. Anatomy of a query request When a querier receives a query range request, it contains the following parameters:\nquery: the PromQL query expression itself (e.g. rate(node_cpu_seconds_total[1m])) start: the start time end: the end time step: the query resolution (e.g. 30 to have 1 resulting data point every 30s) Given a query, the querier analyzes the start and end time range to compute a list of all known blocks containing at least 1 sample within this time range. Given the list of blocks, the querier then computes a list of store-gateway instances holding these blocks and sends a request to each matching store-gateway instance asking to fetch all the samples for the series matching the query within the start and end time range.\nThe request sent to each store-gateway contains the list of block IDs that are expected to be queried, and the response sent back by the store-gateway to the querier contains the list of block IDs that were actually queried. This list may be a subset of the requested blocks, for example due to recent blocks resharding event (ie. last few seconds). The querier runs a consistency check on responses received from the store-gateways to ensure all expected blocks have been queried; if not, the querier retries to fetch samples from missing blocks from different store-gateways (if the -store-gateway.sharding-ring.replication-factor is greater than 1) and if the consistency check fails after all retries, the query execution fails as well (correctness is always guaranteed).\nIf the query time range covers a period within -querier.query-ingesters-within duration, the querier also sends the request to all ingesters, in order to fetch samples that have not been uploaded to the long-term storage yet.\nOnce all samples have been fetched from both store-gateways and ingesters, the querier proceeds with running the PromQL engine to execute the query and send back the result to the client.\nHow queriers connect to store-gateway Queriers need to discover store-gateways in order to connect to them at query time. The service discovery mechanism used depends whether blocks sharding is enabled in the store-gateways.\nWhen blocks sharding is enabled, queriers need to access to the store-gateways hash ring and thus queriers need to be configured with the same -store-gateway.sharding-ring.* flags (or their respective YAML config options) store-gateways have been configured.\nWhen blocks sharding is disabled, queriers need the -querier.store-gateway-addresses CLI flag (or its respective YAML config option) being set to a comma separated list of store-gateway addresses in DNS Service Discovery format. Queriers will evenly balance the requests to query blocks across the resolved addresses.\nCaching The querier supports the following caches:\nMetadata cache Caching is optional, but highly recommended in a production environment. Please also check out the production tips for more information about configuring the cache.\nMetadata cache Store-gateway and querier can use memcached for caching bucket metadata:\nList of tenants List of blocks per tenant Block’s meta.json content Block’s deletion-mark.json existence and content Tenant’s bucket-index.json.gz content Using the metadata cache can significantly reduce the number of API calls to object storage and protects from linearly scale the number of these API calls with the number of querier and store-gateway instances (because the bucket is periodically scanned and synched by each querier and store-gateway).\nTo enable metadata cache, please set -blocks-storage.bucket-store.metadata-cache.backend. Only memcached backend is supported currently. Memcached client has additional configuration available via flags with -blocks-storage.bucket-store.metadata-cache.memcached.* prefix.\nAdditional options for configuring metadata cache have -blocks-storage.bucket-store.metadata-cache.* prefix. By configuring TTL to zero or negative value, caching of given item type is disabled.\nThe same memcached backend cluster should be shared between store-gateways and queriers.\nQuerier configuration This section described the querier configuration. For the general Cortex configuration and references to common config blocks, please refer to the configuration documentation.\nquerier_config The querier_config configures the Cortex querier.\nquerier: # The maximum number of concurrent queries. # CLI flag: -querier.max-concurrent [max_concurrent: \u003cint\u003e | default = 20] # The timeout for a query. # CLI flag: -querier.timeout [timeout: \u003cduration\u003e | default = 2m] # Deprecated (This feature will be always on after v1.18): Use streaming RPCs # for metadata APIs from ingester. # CLI flag: -querier.ingester-metadata-streaming [ingester_metadata_streaming: \u003cboolean\u003e | default = true] # Use LabelNames ingester RPCs with match params. # CLI flag: -querier.ingester-label-names-with-matchers [ingester_label_names_with_matchers: \u003cboolean\u003e | default = false] # Maximum number of samples a single query can load into memory. # CLI flag: -querier.max-samples [max_samples: \u003cint\u003e | default = 50000000] # Maximum lookback beyond which queries are not sent to ingester. 0 means all # queries are sent to ingester. # CLI flag: -querier.query-ingesters-within [query_ingesters_within: \u003cduration\u003e | default = 0s] # Enable returning samples stats per steps in query response. # CLI flag: -querier.per-step-stats-enabled [per_step_stats_enabled: \u003cboolean\u003e | default = false] # Use compression for metrics query API or instant and range query APIs. # Supports 'gzip' and '' (disable compression) # CLI flag: -querier.response-compression [response_compression: \u003cstring\u003e | default = \"gzip\"] # The time after which a metric should be queried from storage and not just # ingesters. 0 means all queries are sent to store. When running the blocks # storage, if this option is enabled, the time range of the query sent to the # store will be manipulated to ensure the query end is not more recent than # 'now - query-store-after'. # CLI flag: -querier.query-store-after [query_store_after: \u003cduration\u003e | default = 0s] # Maximum duration into the future you can query. 0 to disable. # CLI flag: -querier.max-query-into-future [max_query_into_future: \u003cduration\u003e | default = 10m] # The default evaluation interval or step size for subqueries. # CLI flag: -querier.default-evaluation-interval [default_evaluation_interval: \u003cduration\u003e | default = 1m] # Max number of steps allowed for every subquery expression in query. Number # of steps is calculated using subquery range / step. A value \u003e 0 enables it. # CLI flag: -querier.max-subquery-steps [max_subquery_steps: \u003cint\u003e | default = 0] # Active query tracker monitors active queries, and writes them to the file in # given directory. If Cortex discovers any queries in this log during startup, # it will log them to the log file. Setting to empty value disables active # query tracker, which also disables -querier.max-concurrent option. # CLI flag: -querier.active-query-tracker-dir [active_query_tracker_dir: \u003cstring\u003e | default = \"./active-query-tracker\"] # Time since the last sample after which a time series is considered stale and # ignored by expression evaluations. # CLI flag: -querier.lookback-delta [lookback_delta: \u003cduration\u003e | default = 5m] # Comma separated list of store-gateway addresses in DNS Service Discovery # format. This option should be set when using the blocks storage and the # store-gateway sharding is disabled (when enabled, the store-gateway # instances form a ring and addresses are picked from the ring). # CLI flag: -querier.store-gateway-addresses [store_gateway_addresses: \u003cstring\u003e | default = \"\"] store_gateway_client: # Enable TLS for gRPC client connecting to store-gateway. # CLI flag: -querier.store-gateway-client.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -querier.store-gateway-client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -querier.store-gateway-client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. # If not set, the host's root CA certificates are used. # CLI flag: -querier.store-gateway-client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -querier.store-gateway-client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -querier.store-gateway-client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy' and '' (disable compression) # CLI flag: -querier.store-gateway-client.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # EXPERIMENTAL: If enabled, gRPC clients perform health checks for each # target and fail the request if the target is marked as unhealthy. healthcheck_config: # The number of consecutive failed health checks required before # considering a target unhealthy. 0 means disabled. # CLI flag: -querier.store-gateway-client.unhealthy-threshold [unhealthy_threshold: \u003cint\u003e | default = 0] # The approximate amount of time between health checks of an individual # target. # CLI flag: -querier.store-gateway-client.interval [interval: \u003cduration\u003e | default = 5s] # The amount of time during which no response from a target means a failed # health check. # CLI flag: -querier.store-gateway-client.timeout [timeout: \u003cduration\u003e | default = 1s] # If enabled, store gateway query stats will be logged using `info` log level. # CLI flag: -querier.store-gateway-query-stats-enabled [store_gateway_query_stats: \u003cboolean\u003e | default = true] # The maximum number of times we attempt fetching missing blocks from # different store-gateways. If no more store-gateways are left (ie. due to # lower replication factor) than we'll end the retries earlier # CLI flag: -querier.store-gateway-consistency-check-max-attempts [store_gateway_consistency_check_max_attempts: \u003cint\u003e | default = 3] # When distributor's sharding strategy is shuffle-sharding and this setting is # \u003e 0, queriers fetch in-memory series from the minimum set of required # ingesters, selecting only ingesters which may have received series since # 'now - lookback period'. The lookback period should be greater or equal than # the configured 'query store after' and 'query ingesters within'. If this # setting is 0, queriers always query all ingesters (ingesters shuffle # sharding on read path is disabled). # CLI flag: -querier.shuffle-sharding-ingesters-lookback-period [shuffle_sharding_ingesters_lookback_period: \u003cduration\u003e | default = 0s] # Experimental. Use Thanos promql engine # https://github.com/thanos-io/promql-engine rather than the Prometheus promql # engine. # CLI flag: -querier.thanos-engine [thanos_engine: \u003cboolean\u003e | default = false] # If enabled, ignore max query length check at Querier select method. Users # can choose to ignore it since the validation can be done before Querier # evaluation like at Query Frontend or Ruler. # CLI flag: -querier.ignore-max-query-length [ignore_max_query_length: \u003cboolean\u003e | default = false] # [Experimental] If true, experimental promQL functions are enabled. # CLI flag: -querier.enable-promql-experimental-functions [enable_promql_experimental_functions: \u003cboolean\u003e | default = false] blocks_storage_config The blocks_storage_config configures the blocks storage.\nblocks_storage: # Backend storage to use. Supported backends are: s3, gcs, azure, swift, # filesystem. # CLI flag: -blocks-storage.backend [backend: \u003cstring\u003e | default = \"s3\"] s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an # S3-compatible service in hostname:port format. # CLI flag: -blocks-storage.s3.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # S3 region. If unset, the client will issue a S3 GetBucketLocation API call # to autodetect it. # CLI flag: -blocks-storage.s3.region [region: \u003cstring\u003e | default = \"\"] # S3 bucket name # CLI flag: -blocks-storage.s3.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # S3 secret access key # CLI flag: -blocks-storage.s3.secret-access-key [secret_access_key: \u003cstring\u003e | default = \"\"] # S3 access key ID # CLI flag: -blocks-storage.s3.access-key-id [access_key_id: \u003cstring\u003e | default = \"\"] # If enabled, use http:// for the S3 endpoint instead of https://. This # could be useful in local dev/test environments while using an # S3-compatible backend storage, like Minio. # CLI flag: -blocks-storage.s3.insecure [insecure: \u003cboolean\u003e | default = false] # The signature version to use for authenticating against S3. Supported # values are: v4, v2. # CLI flag: -blocks-storage.s3.signature-version [signature_version: \u003cstring\u003e | default = \"v4\"] # The s3 bucket lookup style. Supported values are: auto, virtual-hosted, # path. # CLI flag: -blocks-storage.s3.bucket-lookup-type [bucket_lookup_type: \u003cstring\u003e | default = \"auto\"] # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum # algorithm to verify the provided digest. If false, use CRC32C algorithm # instead. # CLI flag: -blocks-storage.s3.send-content-md5 [send_content_md5: \u003cboolean\u003e | default = true] # The list api version. Supported values are: v1, v2, and ''. # CLI flag: -blocks-storage.s3.list-objects-version [list_objects_version: \u003cstring\u003e | default = \"\"] # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: blocks-storage [sse: \u003cs3_sse_config\u003e] http: # The time an idle connection will remain idle before closing. # CLI flag: -blocks-storage.s3.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -blocks-storage.s3.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -blocks-storage.s3.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -blocks-storage.s3.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully # writing the request headers if the request has an Expect header. 0 to # send the request body immediately. # CLI flag: -blocks-storage.s3.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 # means no limit. # CLI flag: -blocks-storage.s3.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, # a built-in default value is used. # CLI flag: -blocks-storage.s3.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -blocks-storage.s3.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] gcs: # GCS bucket name # CLI flag: -blocks-storage.gcs.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # JSON representing either a Google Developers Console # client_credentials.json file or a Google Developers service account key # file. If empty, fallback to Google default logic. # CLI flag: -blocks-storage.gcs.service-account [service_account: \u003cstring\u003e | default = \"\"] azure: # Azure storage account name # CLI flag: -blocks-storage.azure.account-name [account_name: \u003cstring\u003e | default = \"\"] # Azure storage account key # CLI flag: -blocks-storage.azure.account-key [account_key: \u003cstring\u003e | default = \"\"] # The values of `account-name` and `endpoint-suffix` values will not be # ignored if `connection-string` is set. Use this method over `account-key` # if you need to authenticate via a SAS token or if you use the Azurite # emulator. # CLI flag: -blocks-storage.azure.connection-string [connection_string: \u003cstring\u003e | default = \"\"] # Azure storage container name # CLI flag: -blocks-storage.azure.container-name [container_name: \u003cstring\u003e | default = \"\"] # Azure storage endpoint suffix without schema. The account name will be # prefixed to this value to create the FQDN # CLI flag: -blocks-storage.azure.endpoint-suffix [endpoint_suffix: \u003cstring\u003e | default = \"\"] # Number of retries for recoverable errors # CLI flag: -blocks-storage.azure.max-retries [max_retries: \u003cint\u003e | default = 20] # Deprecated: Azure storage MSI resource. It will be set automatically by # Azure SDK. # CLI flag: -blocks-storage.azure.msi-resource [msi_resource: \u003cstring\u003e | default = \"\"] # Azure storage MSI resource managed identity client Id. If not supplied # default Azure credential will be used. Set it to empty if you need to # authenticate via Azure Workload Identity. # CLI flag: -blocks-storage.azure.user-assigned-id [user_assigned_id: \u003cstring\u003e | default = \"\"] http: # The time an idle connection will remain idle before closing. # CLI flag: -blocks-storage.azure.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -blocks-storage.azure.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -blocks-storage.azure.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -blocks-storage.azure.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully # writing the request headers if the request has an Expect header. 0 to # send the request body immediately. # CLI flag: -blocks-storage.azure.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 # means no limit. # CLI flag: -blocks-storage.azure.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, # a built-in default value is used. # CLI flag: -blocks-storage.azure.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -blocks-storage.azure.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] swift: # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -blocks-storage.swift.auth-version [auth_version: \u003cint\u003e | default = 0] # OpenStack Swift authentication URL # CLI flag: -blocks-storage.swift.auth-url [auth_url: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential ID. # CLI flag: -blocks-storage.swift.application-credential-id [application_credential_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential name. # CLI flag: -blocks-storage.swift.application-credential-name [application_credential_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential secret. # CLI flag: -blocks-storage.swift.application-credential-secret [application_credential_secret: \u003cstring\u003e | default = \"\"] # OpenStack Swift username. # CLI flag: -blocks-storage.swift.username [username: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -blocks-storage.swift.user-domain-name [user_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -blocks-storage.swift.user-domain-id [user_domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user ID. # CLI flag: -blocks-storage.swift.user-id [user_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift API key. # CLI flag: -blocks-storage.swift.password [password: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -blocks-storage.swift.domain-id [domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -blocks-storage.swift.domain-name [domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift project ID (v2,v3 auth only). # CLI flag: -blocks-storage.swift.project-id [project_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift project name (v2,v3 auth only). # CLI flag: -blocks-storage.swift.project-name [project_name: \u003cstring\u003e | default = \"\"] # ID of the OpenStack Swift project's domain (v3 auth only), only needed if # it differs the from user domain. # CLI flag: -blocks-storage.swift.project-domain-id [project_domain_id: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift project's domain (v3 auth only), only needed # if it differs from the user domain. # CLI flag: -blocks-storage.swift.project-domain-name [project_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift Region to use (v2,v3 auth only). # CLI flag: -blocks-storage.swift.region-name [region_name: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift container to put chunks in. # CLI flag: -blocks-storage.swift.container-name [container_name: \u003cstring\u003e | default = \"\"] # Max retries on requests error. # CLI flag: -blocks-storage.swift.max-retries [max_retries: \u003cint\u003e | default = 3] # Time after which a connection attempt is aborted. # CLI flag: -blocks-storage.swift.connect-timeout [connect_timeout: \u003cduration\u003e | default = 10s] # Time after which an idle request is aborted. The timeout watchdog is reset # each time some data is received, so the timeout triggers after X time no # data is received on a request. # CLI flag: -blocks-storage.swift.request-timeout [request_timeout: \u003cduration\u003e | default = 5s] filesystem: # Local filesystem storage directory. # CLI flag: -blocks-storage.filesystem.dir [dir: \u003cstring\u003e | default = \"\"] # This configures how the querier and store-gateway discover and synchronize # blocks stored in the bucket. bucket_store: # Directory to store synchronized TSDB index headers. # CLI flag: -blocks-storage.bucket-store.sync-dir [sync_dir: \u003cstring\u003e | default = \"tsdb-sync\"] # How frequently to scan the bucket, or to refresh the bucket index (if # enabled), in order to look for changes (new blocks shipped by ingesters # and blocks deleted by retention or compaction). # CLI flag: -blocks-storage.bucket-store.sync-interval [sync_interval: \u003cduration\u003e | default = 15m] # Max number of concurrent queries to execute against the long-term storage. # The limit is shared across all tenants. # CLI flag: -blocks-storage.bucket-store.max-concurrent [max_concurrent: \u003cint\u003e | default = 100] # Max number of inflight queries to execute against the long-term storage. # The limit is shared across all tenants. 0 to disable. # CLI flag: -blocks-storage.bucket-store.max-inflight-requests [max_inflight_requests: \u003cint\u003e | default = 0] # Maximum number of concurrent tenants syncing blocks. # CLI flag: -blocks-storage.bucket-store.tenant-sync-concurrency [tenant_sync_concurrency: \u003cint\u003e | default = 10] # Maximum number of concurrent blocks syncing per tenant. # CLI flag: -blocks-storage.bucket-store.block-sync-concurrency [block_sync_concurrency: \u003cint\u003e | default = 20] # Number of Go routines to use when syncing block meta files from object # storage per tenant. # CLI flag: -blocks-storage.bucket-store.meta-sync-concurrency [meta_sync_concurrency: \u003cint\u003e | default = 20] # Minimum age of a block before it's being read. Set it to safe value (e.g # 30m) if your object storage is eventually consistent. GCS and S3 are # (roughly) strongly consistent. # CLI flag: -blocks-storage.bucket-store.consistency-delay [consistency_delay: \u003cduration\u003e | default = 0s] index_cache: # The index cache backend type. Multiple cache backend can be provided as # a comma-separated ordered list to enable the implementation of a cache # hierarchy. Supported values: inmemory, memcached, redis. # CLI flag: -blocks-storage.bucket-store.index-cache.backend [backend: \u003cstring\u003e | default = \"inmemory\"] inmemory: # Maximum size in bytes of in-memory index cache used to speed up blocks # index lookups (shared between all tenants). # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes [max_size_bytes: \u003cint\u003e | default = 1073741824] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items [enabled_items: \u003clist of string\u003e | default = []] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV # query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup # made after that). # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. # If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should # run. If more keys are specified, internally keys are split into # multiple batches and fetched concurrently, honoring the max # concurrency. If set to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items [enabled_items: \u003clist of string\u003e | default = []] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.index-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching # is when data is stored in memory instead of fetching data each time. # See https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items [enabled_items: \u003clist of string\u003e | default = []] multilevel: # The maximum number of concurrent asynchronous operations can occur # when backfilling cache items. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of items to backfill per asynchronous operation. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-backfill-items [max_backfill_items: \u003cint\u003e | default = 10000] chunks_cache: # The chunks cache backend type. Single or Multiple cache backend can be # provided. Supported values in single cache: memcached, redis, inmemory, # and '' (disable). Supported values in multi level cache: a # comma-separated list of (inmemory, memcached, redis) # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend [backend: \u003cstring\u003e | default = \"\"] inmemory: # Maximum size in bytes of in-memory chunk cache used to speed up chunk # lookups (shared between all tenants). # CLI flag: -blocks-storage.bucket-store.chunks-cache.inmemory.max-size-bytes [max_size_bytes: \u003cint\u003e | default = 1073741824] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV # query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup # made after that). # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. # If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should # run. If more keys are specified, internally keys are split into # multiple batches and fetched concurrently, honoring the max # concurrency. If set to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching # is when data is stored in memory instead of fetching data each time. # See https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] multilevel: # The maximum number of concurrent asynchronous operations can occur # when backfilling cache items. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of items to backfill per asynchronous operation. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-backfill-items [max_backfill_items: \u003cint\u003e | default = 10000] # Size of each subrange that bucket object is split into for better # caching. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-size [subrange_size: \u003cint\u003e | default = 16000] # Maximum number of sub-GetRange requests that a single GetRange request # can be split into when fetching chunks. Zero or negative value = # unlimited number of sub-requests. # CLI flag: -blocks-storage.bucket-store.chunks-cache.max-get-range-requests [max_get_range_requests: \u003cint\u003e | default = 3] # TTL for caching object attributes for chunks. # CLI flag: -blocks-storage.bucket-store.chunks-cache.attributes-ttl [attributes_ttl: \u003cduration\u003e | default = 168h] # TTL for caching individual chunks subranges. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-ttl [subrange_ttl: \u003cduration\u003e | default = 24h] metadata_cache: # Backend for metadata cache, if not empty. Supported values: memcached, # redis, and '' (disable). # CLI flag: -blocks-storage.bucket-store.metadata-cache.backend [backend: \u003cstring\u003e | default = \"\"] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV # query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup # made after that). # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. # If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should # run. If more keys are specified, internally keys are split into # multiple batches and fetched concurrently, honoring the max # concurrency. If set to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching # is when data is stored in memory instead of fetching data each time. # See https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # How long to cache list of tenants in the bucket. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl [tenants_list_ttl: \u003cduration\u003e | default = 15m] # How long to cache list of blocks for each tenant. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenant-blocks-list-ttl [tenant_blocks_list_ttl: \u003cduration\u003e | default = 5m] # How long to cache list of chunks for a block. # CLI flag: -blocks-storage.bucket-store.metadata-cache.chunks-list-ttl [chunks_list_ttl: \u003cduration\u003e | default = 24h] # How long to cache information that block metafile exists. Also used for # user deletion mark file. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-exists-ttl [metafile_exists_ttl: \u003cduration\u003e | default = 2h] # How long to cache information that block metafile doesn't exist. Also # used for user deletion mark file. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-doesnt-exist-ttl [metafile_doesnt_exist_ttl: \u003cduration\u003e | default = 5m] # How long to cache content of the metafile. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-content-ttl [metafile_content_ttl: \u003cduration\u003e | default = 24h] # Maximum size of metafile content to cache in bytes. Caching will be # skipped if the content exceeds this size. This is useful to avoid # network round trip for large content if the configured caching backend # has an hard limit on cached items size (in this case, you should set # this limit to the same limit in the caching backend). # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-max-size-bytes [metafile_max_size_bytes: \u003cint\u003e | default = 1048576] # How long to cache attributes of the block metafile. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-attributes-ttl [metafile_attributes_ttl: \u003cduration\u003e | default = 168h] # How long to cache attributes of the block index. # CLI flag: -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl [block_index_attributes_ttl: \u003cduration\u003e | default = 168h] # How long to cache content of the bucket index. # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl [bucket_index_content_ttl: \u003cduration\u003e | default = 5m] # Maximum size of bucket index content to cache in bytes. Caching will be # skipped if the content exceeds this size. This is useful to avoid # network round trip for large content if the configured caching backend # has an hard limit on cached items size (in this case, you should set # this limit to the same limit in the caching backend). # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-max-size-bytes [bucket_index_max_size_bytes: \u003cint\u003e | default = 1048576] # Duration after which the blocks marked for deletion will be filtered out # while fetching blocks. The idea of ignore-deletion-marks-delay is to # ignore blocks that are marked for deletion with some delay. This ensures # store can still serve blocks that are meant to be deleted but do not have # a replacement yet. Default is 6h, half of the default value for # -compactor.deletion-delay. # CLI flag: -blocks-storage.bucket-store.ignore-deletion-marks-delay [ignore_deletion_mark_delay: \u003cduration\u003e | default = 6h] # The blocks created since `now() - ignore_blocks_within` will not be # synced. This should be used together with `-querier.query-store-after` to # filter out the blocks that are too new to be queried. A reasonable value # for this flag would be `-querier.query-store-after - # blocks-storage.bucket-store.bucket-index.max-stale-period` to give some # buffer. 0 to disable. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: \u003cduration\u003e | default = 0s] bucket_index: # True to enable querier and store-gateway to discover blocks in the # storage via bucket index instead of bucket scanning. # CLI flag: -blocks-storage.bucket-store.bucket-index.enabled [enabled: \u003cboolean\u003e | default = false] # How frequently a bucket index, which previously failed to load, should # be tried to load again. This option is used only by querier. # CLI flag: -blocks-storage.bucket-store.bucket-index.update-on-error-interval [update_on_error_interval: \u003cduration\u003e | default = 1m] # How long a unused bucket index should be cached. Once this timeout # expires, the unused bucket index is removed from the in-memory cache. # This option is used only by querier. # CLI flag: -blocks-storage.bucket-store.bucket-index.idle-timeout [idle_timeout: \u003cduration\u003e | default = 1h] # The maximum allowed age of a bucket index (last updated) before queries # start failing because the bucket index is too old. The bucket index is # periodically updated by the compactor, while this check is enforced in # the querier (at query time). # CLI flag: -blocks-storage.bucket-store.bucket-index.max-stale-period [max_stale_period: \u003cduration\u003e | default = 1h] # One of concurrent, recursive, bucket_index. When set to concurrent, stores # will concurrently issue one call per directory to discover active blocks # in the bucket. The recursive strategy iterates through all objects in the # bucket, recursively traversing into each directory. This avoids N+1 calls # at the expense of having slower bucket iterations. bucket_index strategy # can be used in Compactor only and utilizes the existing bucket index to # fetch block IDs to sync. This avoids iterating the bucket but can be # impacted by delays of cleaner creating bucket index. # CLI flag: -blocks-storage.bucket-store.block-discovery-strategy [block_discovery_strategy: \u003cstring\u003e | default = \"concurrent\"] # Max size - in bytes - of a chunks pool, used to reduce memory allocations. # The pool is shared across all tenants. 0 to disable the limit. # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes [max_chunk_pool_bytes: \u003cint\u003e | default = 2147483648] # If enabled, store-gateway will lazily memory-map an index-header only once # required by a query. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-enabled [index_header_lazy_loading_enabled: \u003cboolean\u003e | default = false] # If index-header lazy loading is enabled and this setting is \u003e 0, the # store-gateway will release memory-mapped index-headers after 'idle # timeout' inactivity. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout [index_header_lazy_loading_idle_timeout: \u003cduration\u003e | default = 20m] # If true, Store Gateway will estimate postings size and try to lazily # expand postings if it downloads less data than expanding all postings. # CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled [lazy_expanded_postings_enabled: \u003cboolean\u003e | default = false] # Controls how many series to fetch per batch in Store Gateway. Default # value is 10000. # CLI flag: -blocks-storage.bucket-store.series-batch-size [series_batch_size: \u003cint\u003e | default = 10000] token_bucket_bytes_limiter: # Token bucket bytes limiter mode. Supported values are: disabled, dryrun, # enabled # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.mode [mode: \u003cstring\u003e | default = \"disabled\"] # Instance token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.instance-token-bucket-size [instance_token_bucket_size: \u003cint\u003e | default = 859832320] # User token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.user-token-bucket-size [user_token_bucket_size: \u003cint\u003e | default = 644874240] # Request token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.request-token-bucket-size [request_token_bucket_size: \u003cint\u003e | default = 4194304] tsdb: # Local directory to store TSDBs in the ingesters. # CLI flag: -blocks-storage.tsdb.dir [dir: \u003cstring\u003e | default = \"tsdb\"] # TSDB blocks range period. # CLI flag: -blocks-storage.tsdb.block-ranges-period [block_ranges_period: \u003clist of duration\u003e | default = 2h0m0s] # TSDB blocks retention in the ingester before a block is removed. This # should be larger than the block_ranges_period and large enough to give # store-gateways and queriers enough time to discover newly uploaded blocks. # CLI flag: -blocks-storage.tsdb.retention-period [retention_period: \u003cduration\u003e | default = 6h] # How frequently the TSDB blocks are scanned and new ones are shipped to the # storage. 0 means shipping is disabled. # CLI flag: -blocks-storage.tsdb.ship-interval [ship_interval: \u003cduration\u003e | default = 1m] # Maximum number of tenants concurrently shipping blocks to the storage. # CLI flag: -blocks-storage.tsdb.ship-concurrency [ship_concurrency: \u003cint\u003e | default = 10] # How frequently does Cortex try to compact TSDB head. Block is only created # if data covers smallest block range. Must be greater than 0 and max 30 # minutes. Note that up to 50% jitter is added to the value for the first # compaction to avoid ingesters compacting concurrently. # CLI flag: -blocks-storage.tsdb.head-compaction-interval [head_compaction_interval: \u003cduration\u003e | default = 1m] # Maximum number of tenants concurrently compacting TSDB head into a new # block # CLI flag: -blocks-storage.tsdb.head-compaction-concurrency [head_compaction_concurrency: \u003cint\u003e | default = 5] # If TSDB head is idle for this duration, it is compacted. Note that up to # 25% jitter is added to the value to avoid ingesters compacting # concurrently. 0 means disabled. # CLI flag: -blocks-storage.tsdb.head-compaction-idle-timeout [head_compaction_idle_timeout: \u003cduration\u003e | default = 1h] # The write buffer size used by the head chunks mapper. Lower values reduce # memory utilisation on clusters with a large number of tenants at the cost # of increased disk I/O operations. # CLI flag: -blocks-storage.tsdb.head-chunks-write-buffer-size-bytes [head_chunks_write_buffer_size_bytes: \u003cint\u003e | default = 4194304] # The number of shards of series to use in TSDB (must be a power of 2). # Reducing this will decrease memory footprint, but can negatively impact # performance. # CLI flag: -blocks-storage.tsdb.stripe-size [stripe_size: \u003cint\u003e | default = 16384] # Deprecated (use blocks-storage.tsdb.wal-compression-type instead): True to # enable TSDB WAL compression. # CLI flag: -blocks-storage.tsdb.wal-compression-enabled [wal_compression_enabled: \u003cboolean\u003e | default = false] # TSDB WAL type. Supported values are: 'snappy', 'zstd' and '' (disable # compression) # CLI flag: -blocks-storage.tsdb.wal-compression-type [wal_compression_type: \u003cstring\u003e | default = \"\"] # TSDB WAL segments files max size (bytes). # CLI flag: -blocks-storage.tsdb.wal-segment-size-bytes [wal_segment_size_bytes: \u003cint\u003e | default = 134217728] # True to flush blocks to storage on shutdown. If false, incomplete blocks # will be reused after restart. # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown [flush_blocks_on_shutdown: \u003cboolean\u003e | default = false] # If TSDB has not received any data for this duration, and all blocks from # TSDB have been shipped, TSDB is closed and deleted from local disk. If set # to positive value, this value should be equal or higher than # -querier.query-ingesters-within flag to make sure that TSDB is not closed # prematurely, which could cause partial query results. 0 or negative value # disables closing of idle TSDB. # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout [close_idle_tsdb_timeout: \u003cduration\u003e | default = 0s] # The size of the in-memory queue used before flushing chunks to the disk. # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: \u003cint\u003e | default = 0] # limit the number of concurrently opening TSDB's on startup # CLI flag: -blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup [max_tsdb_opening_concurrency_on_startup: \u003cint\u003e | default = 10] # Deprecated, use maxExemplars in limits instead. If the MaxExemplars value # in limits is set to zero, cortex will fallback on this value. This setting # enables support for exemplars in TSDB and sets the maximum number that # will be stored. 0 or less means disabled. # CLI flag: -blocks-storage.tsdb.max-exemplars [max_exemplars: \u003cint\u003e | default = 0] # True to enable snapshotting of in-memory TSDB data on disk when shutting # down. # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown [memory_snapshot_on_shutdown: \u003cboolean\u003e | default = false] # [EXPERIMENTAL] Configures the maximum number of samples per chunk that can # be out-of-order. # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max [out_of_order_cap_max: \u003cint\u003e | default = 32] # [EXPERIMENTAL] True to enable native histogram. # CLI flag: -blocks-storage.tsdb.enable-native-histograms [enable_native_histograms: \u003cboolean\u003e | default = false] # [EXPERIMENTAL] If enabled, ingesters will cache expanded postings when # querying blocks. Caching can be configured separately for the head and # compacted blocks. expanded_postings_cache: # If enabled, ingesters will cache expanded postings for the head block. # Only queries with with an equal matcher for metric __name__ are cached. head: # Whether the postings cache is enabled or not # CLI flag: -blocks-storage.expanded_postings_cache.head.enabled [enabled: \u003cboolean\u003e | default = false] # Max bytes for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.head.max-bytes [max_bytes: \u003cint\u003e | default = 10485760] # TTL for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.head.ttl [ttl: \u003cduration\u003e | default = 10m] # If enabled, ingesters will cache expanded postings for the compacted # blocks. The cache is shared between all blocks. blocks: # Whether the postings cache is enabled or not # CLI flag: -blocks-storage.expanded_postings_cache.block.enabled [enabled: \u003cboolean\u003e | default = false] # Max bytes for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.block.max-bytes [max_bytes: \u003cint\u003e | default = 10485760] # TTL for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.block.ttl [ttl: \u003cduration\u003e | default = 10m] ","categories":"","description":"","excerpt":" The querier service handles queries using the PromQL query language. …","ref":"/docs/blocks-storage/querier/","tags":"","title":"Querier"},{"body":"The query auditor is a tool bundled in the Cortex repository, but not included in Docker images – this must be built from source. It’s primarily useful for those developing Cortex, but can be helpful to operators as well during certain scenarios (backend migrations come to mind).\nHow it works The query-audit tool performs a set of queries against two backends that expose the Prometheus read API. This is generally the query-frontend component of two Cortex deployments. It will then compare the differences in the responses to determine the average difference for each query. It does this by:\nEnsuring the resulting label sets match. For each label set, ensuring they contain the same number of samples as their pair from the other backend. For each sample, calculates their difference against it’s pair from the other backend/label set. Calculates the average diff per query from the above diffs. Limitations It currently only supports queries with Matrix response types.\nUse cases Correctness testing when working on the read path. Comparing results from different backends. Example Configuration control: host: http://localhost:8080/prometheus headers: \"X-Scope-OrgID\": 1234 test: host: http://localhost:8081/prometheus headers: \"X-Scope-OrgID\": 1234 queries: - query: 'sum(rate(container_cpu_usage_seconds_total[5m]))' start: 2019-11-25T00:00:00Z end: 2019-11-28T00:00:00Z step_size: 15m - query: 'sum(rate(container_cpu_usage_seconds_total[5m])) by (container_name)' start: 2019-11-25T00:00:00Z end: 2019-11-28T00:00:00Z step_size: 15m - query: 'sum(rate(container_cpu_usage_seconds_total[5m])) without (container_name)' start: 2019-11-25T00:00:00Z end: 2019-11-26T00:00:00Z step_size: 15m - query: 'histogram_quantile(0.9, sum(rate(cortex_cache_value_size_bytes_bucket[5m])) by (le, job))' start: 2019-11-25T00:00:00Z end: 2019-11-25T06:00:00Z step_size: 15m # two shardable legs - query: 'sum without (instance, job) (rate(cortex_query_frontend_queue_length[5m])) or sum by (job) (rate(cortex_query_frontend_queue_length[5m]))' start: 2019-11-25T00:00:00Z end: 2019-11-25T06:00:00Z step_size: 15m # one shardable leg - query: 'sum without (instance, job) (rate(cortex_cache_request_duration_seconds_count[5m])) or rate(cortex_cache_request_duration_seconds_count[5m])' start: 2019-11-25T00:00:00Z end: 2019-11-25T06:00:00Z step_size: 15m Example Output Under ideal circumstances, you’ll see output like the following:\n$ go run ./tools/query-audit/ -f config.yaml 0.000000% avg diff for: query: sum(rate(container_cpu_usage_seconds_total[5m])) series: 1 samples: 289 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-28 00:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: sum(rate(container_cpu_usage_seconds_total[5m])) by (container_name) series: 95 samples: 25877 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-28 00:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: sum(rate(container_cpu_usage_seconds_total[5m])) without (container_name) series: 4308 samples: 374989 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-26 00:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: histogram_quantile(0.9, sum(rate(cortex_cache_value_size_bytes_bucket[5m])) by (le, job)) series: 13 samples: 325 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-25 06:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: sum without (instance, job) (rate(cortex_query_frontend_queue_length[5m])) or sum by (job) (rate(cortex_query_frontend_queue_length[5m])) series: 21 samples: 525 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-25 06:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: sum without (instance, job) (rate(cortex_cache_request_duration_seconds_count[5m])) or rate(cortex_cache_request_duration_seconds_count[5m]) series: 942 samples: 23550 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-25 06:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: sum by (namespace) (predict_linear(container_cpu_usage_seconds_total[5m], 10)) series: 16 samples: 400 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-25 06:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: sum by (namespace) (avg_over_time((rate(container_cpu_usage_seconds_total[5m]))[10m:]) \u003e 1) series: 4 samples: 52 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-25 01:00:00 +0000 UTC step: 5m0s ","categories":"","description":"","excerpt":"The query auditor is a tool bundled in the Cortex repository, but not …","ref":"/docs/operations/query-auditor/","tags":"","title":"Query Auditor (tool)"},{"body":"REWE digital builds the technology that drives the e-commerce, app, and food pickup and delivery services for one of Germany’s largest grocery chains, REWE. Like other companies involved in the food supply chain, REWE has seen demand spike during the Covid-19 pandemic. Thanks to its adoption of Cortex last year, the monitoring team has been able to ensure stability at growing scale.\nThe REWE digital subsidiary was started in 2014 to advance its parent company’s digital transformation, so “we have a rather modern tech stack,” says Cloud Platform Engineer Martin Schneppenheim. A lot of the platform is run on Kubernetes using GKE while some is still running on-prem using Nomad. Still, there were some challenges that arose from the Spotify tribe model that the organization adopted. Each of the four tribes at REWE digital – ECOM, FULFILLMENT, CONTENT and PLATFORM, and while the tribes mostly converged on the same technologies, he says, “each platform team has its own solution for, say, Kafka, for Prometheus, for Grafana.”\nPlus, over the past six years, the company has grown from 30 employees to around 600.\nWith this rapid growth, they realized by the end of 2018 that they needed a new solution. The tipping point came when they experienced some out-of-memory issues with Prometheus, he says, “and we had no idea why.”\nEach tribe had one Prometheus HA pair that was used by all the teams in the tribe. One of the tribes used one Prometheus pair which required 30-60 gigabytes of RAM per instance. “We still saw some random out-of-memory kills, and we believe it was because some queries were loading too many samples,” Schneppenheim says. “We had several platform teams doing basically the same thing, and we wanted to tackle such issues organization-wide.”\nSearching for a scalable monitoring solution The solution needed to support the Prometheus format, since all of REWE’s microservices had a Prometheus end point. And the team wanted to have trust in the project’s longevity. After considering M3, Victoria Metrics, and Thanos, the REWE digital team decided to go with Cortex. “Cortex had just been released as a CNCF project, and there were several developers from different companies,” he says. “That was another plus point for us.” The key selling point, he adds, was Cortex’s multi-tenant support, which also involves the different protection mechanisms built into Cortex to limit a tenant’s usage so that a single tenant doesn’t affect the performance for other tenants. “Every platform team was providing one Prometheus for their tribe,” Schneppenheim says, “and we wanted to move to something like a software-as-a-service approach, with just one team that provides Cortex, which can be used by all the teams within the company.”\nImplementation Implementation began with the Big Data tribe, which has since merged with the other tribes and has been the smallest in the company. “We already had Prometheus set up, and we just switched the data source from Prometheus to Cortex,” Schneppenheim says. “In the beginning it was just one dashboard where we switched the data sources, and later on we switched the data source for the whole tribe so that all dashboards used the Cortex data source by default, and the Prometheus deployment basically acted as remote writing Prometheus. We always had the chance to just switch back to the Prometheus, in case there were any failures, so there was not a big risk.”\nIn fact, things went smoothly, and a few months later, the ECOM tribe started writing metrics to Cortex. At the same time, the platform tribe decided to create one Grafana instance and use organizations to offer multi-tenancy. After that second migration, the tribe’s teams were able to migrate dashboards to the new Grafana instance, and then start querying against that data. By the end of the year, all the tribes will have migrated to Cortex and the Grafana instance.\nREWE digital adopted Cortex at “a very early stage,” Schneppenheim says. At first, “sometimes we had to read the code, because there was little documentation, but we were still confident that we took the right decision, because we got lots of support [from the community] in debugging some problems, which were usually misconfigurations.”\nHe points out that configuration has become simpler over the past year, with default values set in v1.0, and more documentation: “Things definitely became better.”\nResults Cortex’s horizontal scaling has proven to be crucial during the Covid-19 pandemic, when REWE’s grocery and food delivery services have seen extremely high demand. “Our primary focus was to ensure stability, so we had to scale, and we deployed more containers,” he says. “That meant we had way more metrics than before, on the one hand, and on the other hand, I believe our developers were watching our dashboards more closely, so we had way more queries as well.”\nSchneppenheim estimates that over the past two months, reads and writes have increased significantly, and the platform was able to handle the added load. Plus, “it was quite easy deploying another set of queriers,” he says.\nAside from that, Schneppenheim says, “the biggest advantage for our company is that we now have a team that can offer one thing as an internal service.” While the tribes’ ops teams still have to manage their own Prometheus servers, they have a much more stable and scalable system. The challenges are unpredictable resource usage on querying and some queries that can load too much data causing Prometheus to OOM, but with Cortex handling all the queries, this is no longer a problem. And while Schneppenheim’s team is still just two people (they’re hiring!), he adds, “we can spend more time actually learning how to run it, and become an expert within the company for Cortex and the things that come along with it, like high cardinality metric series, which we see every two weeks or so. We are the contact for all the monitoring now.”\nThere have been other technical advantages, too: “We have no gaps anymore in our Prometheus and Grafana,” he says. “In case a Prometheus instance fails or if it needs to be restarted, we automatically switch over to the replica with the HA tracker, which is a great thing.”\nWith Cortex’s query results cache, the queries are cached. The REWE digital team has found that this feature makes dashboards “super fast, because the query is likely already cached and it just has to load the new 30 seconds or so, since the last refresh,” says Schneppenheim. “Preloaded dashboards load or refresh really, really fast.”\nPlus, there is a higher retention with Cortex. “We now have 60 days’ retention; we used to have seven days only,” he says.\nThe benefits are also clear as the infrastructure grows. REWE digital has added a few more small Kubernetes clusters, which “obviously have the same monitoring/alerting needs as our biggest clusters,” he says. Previously, the team would have to deploy Prometheus and a separate Grafana instance (along with NGINX and DNS setup).\n“With our new SaaS approach, making monitoring available for these is as easy as adding a Prometheus pair, which sends metrics to our Cortex cluster, and adding this new tenant in our Grafana organization,” he says.\nWith Cortex, they’ve also been able to solve two use cases (Kubernetes clusters that had been split for technical reasons, and cloud migration) that required metrics from two different clusters to be available with the same tenant. “The dev teams had the need to aggregate metrics across these two clusters, which was easily possible, because we just ingested them under the same tenant ID,” says Schneppenheim.\nAnd those out-of-memory issues? “We are constantly growing, not only on the query side, but also on the ingesting metrics side as we onboard teams and tribes,” he says. “But we have fine-tuned it quite well, and there are not as many OOM kills, and if there are, we don’t see them in Grafana. That’s important to us, that our developers have a smooth experience.” (Most tribes use Grafana alerting; one uses Prometheus Alertmanager.)\nLooking ahead REWE digital’s main focus right now is to onboard the rest of the teams to Cortex. But looking ahead, the team is exploring Grafana Cloud Agent for the tribes that aren’t using Prometheus Alertmanager. “They don’t need Prometheus; we only use Prometheus to scrape the targets and send samples to our Cortex,” he says, “so that could definitely be interesting, especially given the performance improvements. Its sole purpose is to send metrics to Cortex as our remote-write backend, so maybe there will be other advantages in the future, like a more close monitoring.”\nThe in-development Cortex Blocks storage engine is also interesting to the team as a solution for the bottleneck it has around a small BigTable cluster. “We just run three BigTable nodes, and the BigTable read latency sometimes peaks at one second, which is also the upper bucket limit in the histogram,” he says. “This happens if users open Grafana dashboards querying a long time range with many panels. Our hope is that switching from BigTable to the new storage engine would fix this as the object store (GCS) scales on-demand.”\nThe REWE digital team has built and open sourced its own Cortex gateway, which is on the project roadmap. “This might be a chance for us to contribute,” says Schneppenheim.\nSchneppenheim is also hopeful that the positive results REWE digital has seen with Cortex may lead to its further adoption throughout the greater REWE Group organization. “We’re just a small company within the REWE group,” he says, but “we might offer it as internal software as a service for other parts of the Group. They can trust our solution.”\n","categories":"","description":"","excerpt":"REWE digital builds the technology that drives the e-commerce, app, …","ref":"/docs/case-studies/rewe-digital/","tags":"","title":"How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic"},{"body":"The blocks storage is a Cortex storage engine based on Prometheus TSDB: it stores each tenant’s time series into their own TSDB which write out their series to a on-disk block (defaults to 2h block range periods). Each block is composed by chunk files - containing the timestamp-value pairs for multiple series - and an index, which indexes metric names and labels to time series in the chunk files.\nThe supported backends for the blocks storage are:\nAmazon S3 Google Cloud Storage Microsoft Azure Storage OpenStack Swift Local Filesystem (single node only) Internally, some components are based on Thanos, but no Thanos knowledge is required in order to run it.\nArchitecture When running the Cortex blocks storage, the Cortex architecture doesn’t significantly change and thus the general architecture documentation applies to the blocks storage as well. However, there are two additional Cortex services when running the blocks storage:\nStore-gateway Compactor The store-gateway is responsible to query blocks and is used by the querier at query time. The store-gateway is required when running the blocks storage.\nThe compactor is responsible to merge and deduplicate smaller blocks into larger ones, in order to reduce the number of blocks stored in the long-term storage for a given tenant and query them more efficiently. It also keeps the bucket index updated and, for this reason, it’s a required component.\nThe alertmanager and ruler components can also use object storage to store its configurations and rules uploaded by users. In that case a separate bucket should be created to store alertmanager configurations and rules: using the same bucket between ruler/alertmanager and blocks will cause issue with the compactor.\nThe write path Ingesters receive incoming samples from the distributors. Each push request belongs to a tenant, and the ingester appends the received samples to the specific per-tenant TSDB stored on the local disk. The received samples are both kept in-memory and written to a write-ahead log (WAL) and used to recover the in-memory series in case the ingester abruptly terminates. The per-tenant TSDB is lazily created in each ingester as soon as the first samples are received for that tenant.\nThe in-memory samples are periodically flushed to disk - and the WAL truncated - when a new TSDB block is created, which by default occurs every 2 hours. Each newly created block is then uploaded to the long-term storage and kept in the ingester until the configured -blocks-storage.tsdb.retention-period expires, in order to give queriers and store-gateways enough time to discover the new block on the storage and download its index-header.\nIn order to effectively use the WAL and being able to recover the in-memory series upon ingester abruptly termination, the WAL needs to be stored to a persistent disk which can survive in the event of an ingester failure (ie. AWS EBS volume or GCP persistent disk when running in the cloud). For example, if you’re running the Cortex cluster in Kubernetes, you may use a StatefulSet with a persistent volume claim for the ingesters. The location on the filesystem where the WAL is stored is the same where local TSDB blocks (compacted from head) are stored and cannot be decoupled. See also the timeline of block uploads and disk space estimate.\nDistributor series sharding and replication The series sharding and replication done by the distributor doesn’t change based on the storage engine.\nIt’s important to note that due to the replication factor N (typically 3), each time series is stored by N ingesters. Since each ingester writes its own block to the long-term storage, this leads a storage utilization N times more. Compactor solves this problem by merging blocks from multiple ingesters into a single block, and removing duplicated samples. After blocks compaction, the storage utilization is significantly smaller.\nFor more information, please refer to the following dedicated sections:\nCompactor Production tips The read path Queriers and store-gateways periodically iterate over the storage bucket to discover blocks recently uploaded by ingesters.\nFor each discovered block, queriers only download the block’s meta.json file (containing some metadata including min and max timestamp of samples within the block), while store-gateways download the meta.json as well as the index-header, which is a small subset of the block’s index used by the store-gateway to lookup series at query time.\nQueriers use the blocks metadata to compute the list of blocks that need to be queried at query time and fetch matching series from the store-gateway instances holding the required blocks.\nFor more information, please refer to the following dedicated sections:\nQuerier Store-gateway Production tips Known issues GitHub issues tagged with the storage/blocks label are the best source of currently known issues affecting the blocks storage.\n","categories":"","description":"","excerpt":"The blocks storage is a Cortex storage engine based on Prometheus …","ref":"/docs/blocks-storage/","tags":"","title":"Blocks Storage"},{"body":"Buoyant, the creator of Linkerd, has had a close relationship with Prometheus and related technologies for years. As of today, that relationship now includes Cortex.\nLinkerd is an open source service mesh for Kubernetes, and part of the CNCF, along with Prometheus and Cortex. Linkerd provides three key classes of features: observability, reliability, and security—all without requiring any changes to your code. That first pillar, observability, has motivated deep Prometheus integration, and is the focus of this case study.\nLinkerd x Prometheus In 2016, Linkerd 1.0 shipped with a visualization module built with Prometheus and Grafana. With the release of Linkerd 2.0 in 2018, Prometheus and Grafana were promoted to first-class components. That evolution was documented in a blog post entitled Prometheus the Right Way, and a KubeCon North America 2018 talk entitled Linkerd 2.0, Now with Extra Prometheus, both co-authored by Frederic Branczyk of Polar Signals and Andrew Seigner of Buoyant.\nThis deep integration in Linkerd 2.0 provided out-of-the-box golden metrics, that is, success rate, request rate, and latency, across all your Kubernetes workloads.\nLinkerd 2.0 with integrated Prometheus and Grafana, out-of-the-box golden metrics\nBuoyant x Cortex Building on the success of Linkerd, Buoyant has created Buoyant Cloud, a global platform health dashboard for Kubernetes. Leveraging Linkerd’s integration with Prometheus, Buoyant Cloud rolls up Linkerd metrics across all Kubernetes clusters to provide global platform-wide observability, including advanced features such as cross-cluster Service Level Objectives for success rate and latency.\nService Level Objectives in Buoyant Cloud\nBuoyant Cloud Prototyping with Prometheus To enable observability in Buoyant Cloud, customers install an agent into their Kubernetes clusters. That agent gathers metrics from Linkerd and sends them up to Buoyant Cloud. Early prototypes of Buoyant Cloud received these metrics, wrote them to pushgateway, and then our own larger Prometheus would scrape metrics from there. While this had the nice property that Linkerd metrics looked the same in Buoyant Cloud as they did in individual Linkerd clusters, gathering metrics across all customers into a single Prometheus instance created an inherent scaling limitation, and a single-point-of-failure.\nObservability Requirements Thinking beyond our early prototypes, we came up with four core requirements for our observability system:\nScalable - To support all Kubernetes clusters across our growing customer base, we needed a system that could scale as-needed. Bonus points if we could scale our reads and writes independently. Reliable - Users of Linkerd expect nothing less. Multi-tenant - To provide an extra layer of security and performance isolation between customers. Prometheus-compatible - To give our customers a familiar API, and to allow a drop-in replacement for our prototype Prometheus instance. Those first two requirements in particular are not compatible with a single instance of anything. Even Prometheus, which does exceptionally well as a single instance, is intentionally not designed as a distributed, scalable system.\nTake a look at those requirements again, then go to the Cortex website:\nhttps://cortexmetrics.io/\nThis was not intentional, but it was a fortuitous coincidence. Cortex’s one-line description of itself literally satisfied all four of our requirements for Buoyant Cloud.\nBuoyant Cloud Production-Ready with Cortex We set about building a proof of concept to validate whether Cortex could be a viable replacement for Buoyant Cloud’s prototype Prometheus-based observability.\nDevelopment environment The very first attempt at Cortex integration in our development environment was surprisingly smooth, thanks largely to Cortex’s single instance, single process mode. This enabled our developers to operate all of Cortex’s services as a single process in our docker-compose development environment, with no dependencies.\n(Almost) a drop-in replacement Cortex provides a Prometheus-compatible API. To enable multi-tenancy, you must set a X-Scope-OrgID header on every request with a unique identifier for the tenant. We already had a unique public identifier for each Buoyant Cloud customer, so that was a natural fit for this header. Modifying all reads and writes to Prometheus to set this header was a relatively straightforward code change, considering we were about to completely swap out a back-end storage system.\nBlocks Storage Being the creators of Linkerd, we care a lot about operational complexity. Thus, it was no surprise that we wanted to minimize operational complexity and cost for ourselves in maintaining Buoyant Cloud. Cortex’s Chunk Storage was a concern here. Operating our own Cassandra cluster would incur developer time, and paying for DynamoDB, BigTable, or Azure Cosmos DB would incur cost. Cortex’s Blocks Storage removed this complexity and cost by relying on a simple and cheap object store, such as S3, GCS, or Azure Storage. At the time, Blocks Storage was still marked experimental. That’s when we hopped into a Q\u0026A at a KubeCon EU 2020 talk, Scaling Prometheus: How We Got Some Thanos Into Cortex by Thor Hansen of HashiCorp and Marco Pracucci of Grafana Labs. We asked the Cortex maintainers how close to General Availability they felt Blocks Storage was. While wisely not guaranteeing a date, they hinted it may be ready by the next Cortex release (and that hint turned out to be true, Blocks Storage was marked stable in Cortex 1.4.0!). This gave us enough confidence to build our proof of concept around Blocks Storage.\nRecording Rules Buoyant Cloud uses recording rules for important success rate and latency metrics. One unexpected challenge was that Cortex’s multi-tenancy applies to recording rules. This meant that for every customer, we needed to push an identical set of recording rules to the Cortex Ruler API. We ended up writing a small background process that continually checks our database for new customers, and pushes recording rules when they appear.\nCortex in Prod: Writes, then reads Once all the pieces were in place, we enabled production writes to our new Cortex cluster. We continued writing to and reading from our existing Prometheus. Writing to Cortex and Prometheus simultaneously enabled validation of three things. We could 1) Evaluate Cortex’s read performance under full production write load, 2) ensure Cortex query results matched Prometheus, and 3) gather enough historical metrics in Cortex to minimize data loss for our customers.\nWhen we were satisfied with all three of these validation steps, we switched all reads to Cortex. This was the smallest code change in the entire migration. We simply swapped out a few command-line flags pointing to prometheus:9009 with query-frontend.cortex.svc.cluster.local:9009/api/prom. Boom! It all worked!\nThe moment we turned on Cortex reads in production (numbers do not reflect full production load).\nLooking Ahead Our mission with Buoyant Cloud is to enable our customers to build observable, scalable, and secure Kubernetes platforms, without the complexity of stitching together lots of components.\nUsers of Linkerd expect their observability to “just work”, and expect it to be something they are already familiar with and have tools to integrate with. For Linkerd running on a single Kubernetes cluster, Prometheus fits the bill perfectly. For Buoyant Cloud, we believe Cortex can deliver that same familiarity and integration story for all Kubernetes clusters across all customers.\n","categories":"","description":"","excerpt":"Buoyant, the creator of Linkerd, has had a close relationship with …","ref":"/docs/case-studies/buoyant-cloud/","tags":"","title":"Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus"},{"body":"The Cortex documentation is compiled into a website published at cortexmetrics.io. These instructions explain how to run the website locally, in order to have a quick feedback loop while contributing to the documentation or website styling.\nInitial setup The following initial setup is required only once:\nInstall Hugo (extended version) Look for value of HUGO_VERSION in build-image/Dockerfile. Install Node.js v14 or above (alternatively via nvm) Install required Node modules with: cd website \u0026\u0026 npm install \u0026\u0026 cd - Install embedmd v1.0.0: go install github.com/campoy/embedmd@v1.0.0 Run make BUILD_IN_CONTAINER=false web-build Run it Once the initial setup is completed, you can run the website with the following command. The local website will run at http://localhost:1313/\n# Keep this running make web-serve Whenever you change the content of docs/ or markdown files in the repository root / you should run:\nmake BUILD_IN_CONTAINER=false web-pre Whenever you change the config file or CLI flags in the Cortex code, you should rebuild the config file reference documentation:\nmake BUILD_IN_CONTAINER=false doc web-pre ","categories":"","description":"","excerpt":"The Cortex documentation is compiled into a website published at …","ref":"/docs/contributing/how-to-run-the-website-locally/","tags":"","title":"How to run the website locally"},{"body":"You can use the Cortex query frontend with any Prometheus-API compatible service, including Prometheus and Thanos. Use this config file to get the benefits of query parallelisation and caching.\n# Disable the requirement that every request to Cortex has a # X-Scope-OrgID header. `fake` will be substituted in instead. auth_enabled: false # We only want to run the query-frontend module. target: query-frontend # We don't want the usual /api/prom prefix. http_prefix: server: http_listen_port: 9091 query_range: split_queries_by_interval: 24h align_queries_with_step: true cache_results: true # list of request headers forwarded by query frontend to downstream queriers. forward_headers_list: - Authorization results_cache: cache: # We're going to use the in-process \"FIFO\" cache, but you can enable # memcached below. enable_fifocache: true fifocache: size: 1024 validity: 24h # If you want to use a memcached cluster, you can either configure a # headless service in Kubernetes and Cortex will discover the individual # instances using a SRV DNS query (host) or list comma separated # memcached addresses. # host + service: this is the config you should set when you use the # SRV DNS (this is considered stable) # addresses: this is experimental and supports service discovery # (https://cortexmetrics.io/docs/configuration/arguments/#dns-service-discovery) # so it could either be a list of single addresses, or a SRV record # prefixed with dnssrvnoa+. Cortex will then do client-side hashing to # spread the load evenly. # memcached: # expiration : 24h # memcached_client: # host: memcached.default.svc.cluster.local # service: memcached # addresses: \"\" # consistent_hash: true frontend: log_queries_longer_than: 1s compress_responses: true # The query endpoint URL of a Prometheus-API compatible service to which the query-frontend should connect to. downstream_url: http://prometheus.mydomain.com ","categories":"","description":"","excerpt":"You can use the Cortex query frontend with any Prometheus-API …","ref":"/docs/configuration/prometheus-frontend/","tags":"","title":"Prometheus Frontend"},{"body":"The query-tee is a standalone service which can be used for testing purposes to compare the query performances of 2+ backend systems (ie. Cortex clusters) ingesting the same exact series.\nThis service exposes Prometheus-compatible read API endpoints and, for each received request, performs the request against all backends tracking the response time of each backend and then sends back to the client one of the received responses.\nHow to run it You can run query-tee in two ways:\nBuild it from sources go run ./cmd/query-tee -help Run it via the provided Docker image docker run quay.io/cortexproject/query-tee -help The service requires at least 1 backend endpoint (but 2 are required in order to compare performances) configured as comma-separated HTTP(S) URLs via the CLI flag -backend.endpoints. For each incoming request, query-tee will clone the request and send it to each backend, tracking performance metrics for each backend before sending back the response to the client.\nHow it works API endpoints The following Prometheus API endpoints are supported by query-tee:\n/api/v1/query (GET) /api/v1/query_range (GET) /api/v1/labels (GET) /api/v1/label/{name}/values (GET) /api/v1/series (GET) /api/v1/metadata (GET) /api/v1/alerts (GET) /api/v1/rules (GET) Pass-through requests query-tee supports acting as a transparent proxy for requests to routes not matching any of the documented API endpoints above. When enabled, those requests are passed on to just the configured preferred backend. To activate this feature it requires setting -proxy.passthrough-non-registered-routes=true flag and configuring a preferred backend.\nAuthentication query-tee supports HTTP basic authentication. It allows either to configure username and password in the backend URL, to forward the request auth to the backend or merge the two.\nThe request sent from the query-tee to the backend includes HTTP basic authentication when one of the following conditions are met:\nIf the endpoint URL has username and password, query-tee uses it. If the endpoint URL has username only, query-tee keeps the username and inject the password received in the incoming request (if any). If the endpoint URL has no username and no password, query-tee forwards the incoming request basic authentication (if any). Backend response selection query-tee allows to configure a preferred backend from which picking the response to send back to the client. The preferred backend can be configured via the CLI flag -backend.preferred=\u003chostname\u003e, setting it to the hostname of the preferred backend.\nWhen a preferred backend is set, query-tee sends back to the client:\nThe preferred backend response if the status code is 2xx or 4xx Otherwise, the first received 2xx or 4xx response if at least a backend succeeded Otherwise, the first received response When a preferred backend is not set, query-tee sends back to the client:\nThe first received 2xx or 4xx response if at least a backend succeeded Otherwise, the first received response Note: from the query-tee perspective, a backend request is considered successful even if the status code is 4xx because it generally means the error is due to an invalid request and not to a backend issue.\nBackend results comparison query-tee allows to optionally enable the query results comparison between two backends. The results comparison can be enabled via the CLI flag -proxy.compare-responses=true and requires exactly two configured backends with a preferred one.\nWhen the comparison is enabled, the query-tee compares the response received from the two configured backends and logs a message for each query whose results don’t match, as well as keeps track of the number of successful and failed comparison through the metric cortex_querytee_responses_compared_total.\nFloating point sample values are compared with a small tolerance that can be configured via -proxy.value-comparison-tolerance. This prevents false positives due to differences in floating point values rounding introduced by the non deterministic series ordering within the Prometheus PromQL engine.\nSlow backends query-tee sends back to the client the first viable response as soon as available, without waiting to receive a response from all backends.\nExported metrics query-tee exposes the following Prometheus metrics on the port configured via the CLI flag -server.metrics-port:\n# HELP cortex_querytee_request_duration_seconds Time (in seconds) spent serving HTTP requests. # TYPE cortex_querytee_request_duration_seconds histogram cortex_querytee_request_duration_seconds_bucket{backend=\"\u003chostname\u003e\",method=\"\u003cmethod\u003e\",route=\"\u003croute\u003e\",status_code=\"\u003cstatus\u003e\",le=\"\u003cbucket\u003e\"} cortex_querytee_request_duration_seconds_sum{backend=\"\u003chostname\u003e\",method=\"\u003cmethod\u003e\",route=\"\u003croute\u003e\",status_code=\"\u003cstatus\u003e\"} cortex_querytee_request_duration_seconds_count{backend=\"\u003chostname\u003e\",method=\"\u003cmethod\u003e\",route=\"\u003croute\u003e\",status_code=\"\u003cstatus\u003e\"} # HELP cortex_querytee_responses_total Total number of responses sent back to the client by the selected backend. # TYPE cortex_querytee_responses_total counter cortex_querytee_responses_total{backend=\"\u003chostname\u003e\",method=\"\u003cmethod\u003e\",route=\"\u003croute\u003e\"} # HELP cortex_querytee_responses_compared_total Total number of responses compared per route name by result. # TYPE cortex_querytee_responses_compared_total counter cortex_querytee_responses_compared_total{route=\"\u003croute\u003e\",result=\"\u003csuccess|fail\u003e\"} ","categories":"","description":"","excerpt":"The query-tee is a standalone service which can be used for testing …","ref":"/docs/operations/query-tee/","tags":"","title":"Query Tee (service)"},{"body":"Because Cortex is designed to run multiple instances of each component (ingester, querier, etc.), you probably want to automate the placement and shepherding of these instances. Most users choose Kubernetes to do this, but this is not mandatory.\nConfiguration Resource requests If using Kubernetes, each container should specify resource requests so that the scheduler can place them on a node with sufficient capacity.\nFor example, an ingester might request:\nresources: requests: cpu: 4 memory: 10Gi The specific values here should be adjusted based on your own experiences running Cortex - they are very dependent on the rate of data arriving and other factors such as series churn.\nTake extra care with ingesters Ingesters hold hours of timeseries data in memory; you can configure Cortex to replicate the data, but you should take steps to avoid losing all replicas at once:\nDon’t run multiple ingesters on the same node. Don’t run ingesters on preemptible/spot nodes. Spread out ingesters across racks / availability zones / whatever applies in your datacenters. You can ask Kubernetes to avoid running on the same node like this:\naffinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - key: name operator: In values: - ingester topologyKey: \"kubernetes.io/hostname\" Give plenty of time for an ingester to hand over or flush data to store when shutting down; for Kubernetes, this looks like:\nterminationGracePeriodSeconds: 2400 Ask Kubernetes to limit rolling updates to one ingester at a time and signal the old one to stop before the new one is ready:\nstrategy: rollingUpdate: maxSurge: 0 maxUnavailable: 1 Ingesters provide an HTTP hook to signal readiness when all is well; this is valuable because it stops a rolling update at the first problem:\nreadinessProbe: httpGet: path: /ready port: 80 We do not recommend configuring a liveness probe on ingesters - killing them is a last resort and should not be left to a machine.\n","categories":"","description":"","excerpt":"Because Cortex is designed to run multiple instances of each component …","ref":"/docs/guides/running-cortex-on-kubernetes/","tags":"","title":"Running Cortex on Kubernetes"},{"body":" The store-gateway is the Cortex service responsible to query series from blocks. The store-gateway is required when running the Cortex blocks storage.\nThe store-gateway is semi-stateful.\nHow it works The store-gateway needs to have an almost up-to-date view over the storage bucket, in order to discover blocks belonging to their shard. The store-gateway can keep the bucket view updated in to two different ways:\nPeriodically scanning the bucket (default) Periodically downloading the bucket index Bucket index disabled (default) At startup store-gateways iterate over the entire storage bucket to discover blocks for all tenants and download the meta.json and index-header for each block. During this initial bucket synchronization phase, the store-gateway /ready readiness probe endpoint will fail.\nWhile running, store-gateways periodically rescan the storage bucket to discover new blocks (uploaded by the ingesters and compactor) and blocks marked for deletion or fully deleted since the last scan (as a result of compaction). The frequency at which this occurs is configured via -blocks-storage.bucket-store.sync-interval.\nThe blocks chunks and the entire index are never fully downloaded by the store-gateway. The index-header is stored to the local disk, in order to avoid to re-download it on subsequent restarts of a store-gateway. For this reason, it’s recommended - but not required - to run the store-gateway with a persistent disk. For example, if you’re running the Cortex cluster in Kubernetes, you may use a StatefulSet with a persistent volume claim for the store-gateways.\nFor more information about the index-header, please refer to Binary index-header documentation.\nBucket index enabled When bucket index is enabled, the overall workflow is the same but, instead of iterating over the bucket objects, the store-gateway fetch the bucket index for each tenant belonging to their shard in order to discover each tenant’s blocks and block deletion marks.\nFor more information about the bucket index, please refer to bucket index documentation.\nBlocks sharding and replication The store-gateway optionally supports blocks sharding. Sharding can be used to horizontally scale blocks in a large cluster without hitting any vertical scalability limit.\nWhen sharding is enabled, store-gateway instances builds an hash ring and blocks get sharded and replicated across the pool of store-gateway instances registered within the ring.\nStore-gateways continuously monitor the ring state and whenever the ring topology changes (e.g. a new instance has been added/removed or gets healthy/unhealthy) each store-gateway instance resync the blocks assigned to its shard, based on the block ID hash matching the token ranges assigned to the instance itself within the ring.\nFor each block belonging to a store-gateway shard, the store-gateway loads its meta.json, the deletion-mark.json and the index-header. Once a block is loaded on the store-gateway, it’s ready to be queried by queriers. When the querier queries blocks through a store-gateway, the response will contain the list of actually queried block IDs. If a querier tries to query a block which has not been loaded by a store-gateway, the querier will either retry on a different store-gateway (if blocks replication is enabled) or fail the query.\nBlocks can be replicated across multiple store-gateway instances based on a replication factor configured via -store-gateway.sharding-ring.replication-factor. The blocks replication is used to protect from query failures caused by some blocks not loaded by any store-gateway instance at a given time like, for example, in the event of a store-gateway failure or while restarting a store-gateway instance (e.g. during a rolling update).\nThis feature can be enabled via -store-gateway.sharding-enabled=true and requires the backend hash ring to be configured via -store-gateway.sharding-ring.* flags (or their respective YAML config options).\nSharding strategies The store-gateway supports two sharding strategies:\ndefault shuffle-sharding zone-stable-shuffle-sharding The default sharding strategy spreads the blocks of each tenant across all store-gateway instances. It’s the easiest form of sharding supported, but doesn’t provide any workload isolation between different tenants.\nThe shuffle-sharding strategy spreads the blocks of a tenant across a subset of store-gateway instances. This way, the number of store-gateway instances loading blocks of a single tenant is limited and the blast radius of any issue that could be introduced by the tenant’s workload is limited to its shard instances.\nThe shuffle sharding strategy can be enabled via -store-gateway.sharding-strategy=shuffle-sharding and requires the -store-gateway.tenant-shard-size flag (or their respective YAML config options) to be set to the default shard size, which is the default number of store-gateway instances each tenant should be sharded to. The shard size can then be overridden on a per-tenant basis setting the store_gateway_tenant_shard_size in the limits overrides.\nThe zone-stable-shuffle-sharding strategy achieves the same as the shuffle-sharding strategy, but using a different sharding algorithm. The new sharding algorithm ensures that when zone awareness is enabled, when shard size increases or decreases by one, the replicas for any block should only change at most by one instance. This is important for querying store gateway because a block can be retried at most 3 times.\nZone stable shuffle sharding can be enabled via -store-gateway.sharding-ring.zone-stable-shuffle-sharding CLI flag.\nIt will become the default shuffle sharding strategy for store gateway in v1.17.0 release and the previous shuffle sharding algorithm will be removed in v1.18.0 release.\nPlease check out the shuffle sharding documentation for more information about how it works.\nAuto-forget When a store-gateway instance cleanly shutdowns, it automatically unregisters itself from the ring. However, in the event of a crash or node failure, the instance will not be unregistered from the ring, potentially leaving a spurious entry in the ring forever.\nTo protect from this, when an healthy store-gateway instance finds another instance in the ring which is unhealthy for more than 10 times the configured -store-gateway.sharding-ring.heartbeat-timeout, the healthy instance forcibly removes the unhealthy one from the ring.\nThis feature is called auto-forget and is built into the store-gateway.\nZone-awareness The store-gateway replication optionally supports zone-awareness. When zone-aware replication is enabled and the blocks replication factor is \u003e 1, each block is guaranteed to be replicated across store-gateway instances running in different availability zones.\nTo enable the zone-aware replication for the store-gateways you should:\nConfigure the availability zone for each store-gateway via the -store-gateway.sharding-ring.instance-availability-zone CLI flag (or its respective YAML config option) Enable blocks zone-aware replication via the -store-gateway.sharding-ring.zone-awareness-enabled CLI flag (or its respective YAML config option). Please be aware this configuration option should be set to store-gateways, queriers and rulers. Rollout store-gateways, queriers and rulers to apply the new configuration Waiting for stable ring at startup In the event of a cluster cold start or scale up of 2+ store-gateway instances at the same time we may end up in a situation where each new store-gateway instance starts at a slightly different time and thus each one runs the initial blocks sync based on a different state of the ring. For example, in case of a cold start, the first store-gateway joining the ring may load all blocks since the sharding logic runs based on the current state of the ring, which is 1 single store-gateway.\nTo reduce the likelihood this could happen, the store-gateway waits for a stable ring at startup. A ring is considered stable if no instance is added/removed to the ring for at least -store-gateway.sharding-ring.wait-stability-min-duration. If the ring keep getting changed after -store-gateway.sharding-ring.wait-stability-max-duration, the store-gateway will stop waiting for a stable ring and will proceed starting up normally.\nTo disable this waiting logic, you can start the store-gateway with -store-gateway.sharding-ring.wait-stability-min-duration=0.\nBlocks index-header The index-header is a subset of the block index which the store-gateway downloads from the object storage and keeps on the local disk in order to speed up queries.\nAt startup, the store-gateway downloads the index-header of each block belonging to its shard. A store-gateway is not ready until this initial index-header download is completed. Moreover, while running, the store-gateway periodically looks for newly uploaded blocks in the storage and downloads the index-header for the blocks belonging to its shard.\nIndex-header lazy loading By default, each index-header is memory mapped by the store-gateway right after downloading it. In a cluster with a large number of blocks, each store-gateway may have a large amount of memory mapped index-headers, regardless how frequently they’re used at query time.\nCortex supports a configuration option -blocks-storage.bucket-store.index-header-lazy-loading-enabled=true to enable index-header lazy loading. When enabled, index-headers will be memory mapped only once required by a query and will be automatically released after -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout time of inactivity.\nCaching The store-gateway supports the following caches:\nIndex cache Chunks cache Metadata cache Caching is optional, but highly recommended in a production environment. Please also check out the production tips for more information about configuring the cache.\nIndex cache The store-gateway can use a cache to speed up lookups of postings and series from TSDB blocks indexes. Two backends are supported:\ninmemory memcached redis In-memory index cache The inmemory index cache is enabled by default and its max size can be configured through the flag -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes (or config file). The trade-off of using the in-memory index cache is:\nPros: zero latency Cons: increased store-gateway memory usage, not shared across multiple store-gateway replicas (when sharding is disabled or replication factor \u003e 1) Memcached index cache The memcached index cache allows to use Memcached as cache backend. This cache backend is configured using -blocks-storage.bucket-store.index-cache.backend=memcached and requires the Memcached server(s) addresses via -blocks-storage.bucket-store.index-cache.memcached.addresses (or config file). The addresses are resolved using the DNS service provider.\nThe trade-off of using the Memcached index cache is:\nPros: can scale beyond a single node memory (Memcached cluster), shared across multiple store-gateway instances Cons: higher latency in the cache round trip compared to the in-memory one The Memcached client uses a jump hash algorithm to shard cached entries across a cluster of Memcached servers. For this reason, you should make sure memcached servers are not behind any kind of load balancer and their address is configured so that servers are added/removed to the end of the list whenever a scale up/down occurs.\nFor example, if you’re running Memcached in Kubernetes, you may:\nDeploy your Memcached cluster using a StatefulSet Create a headless service for Memcached StatefulSet Configure the Cortex’s Memcached client address using the dnssrvnoa+ service discovery Redis index cache The redis index cache allows to use Redis as cache backend. This cache backend is configured using -blocks-storage.bucket-store.index-cache.backend=redis and requires the Redis server(s) addresses via -blocks-storage.bucket-store.index-cache.redis.addresses (or config file).\nUsing redis as the cache backend has similar trade-offs as using memcached cache backend. However, client side caching can be enabled when using redis backend to avoid Store Gateway fetching data from cache each time. See here for more info and it can be enabled by setting flag -blocks-storage.bucket-store.index-cache.redis.cache-size \u003e 0.\nChunks cache Store-gateway can also use a cache for storing chunks fetched from the storage. Chunks contain actual samples, and can be reused if user query hits the same series for the same time range.\nTo enable chunks cache, please set -blocks-storage.bucket-store.chunks-cache.backend. Chunks can be stored into Memcached or Redis cache. Memcached client can be configured via flags with -blocks-storage.bucket-store.chunks-cache.memcached.* prefix. Redis client can be configured via flags with -blocks-storage.bucket-store.chunks-cache.redis.* prefix.\nThere are additional low-level options for configuring chunks cache. Please refer to other flags with -blocks-storage.bucket-store.chunks-cache.* prefix.\nMetadata cache Store-gateway and querier can use memcached or redis for caching bucket metadata:\nList of tenants List of blocks per tenant Block’s meta.json content Block’s deletion-mark.json existence and content Tenant’s bucket-index.json.gz content Using the metadata cache can significantly reduce the number of API calls to object storage and protects from linearly scale the number of these API calls with the number of querier and store-gateway instances (because the bucket is periodically scanned and synched by each querier and store-gateway).\nTo enable metadata cache, please set -blocks-storage.bucket-store.metadata-cache.backend. memcached and redis backend are supported currently. Memcached client has additional configuration available via flags with -blocks-storage.bucket-store.metadata-cache.memcached.* prefix. Redis client has additional configuration available via flags with -blocks-storage.bucket-store.metadata-cache.redis.* prefix.\nAdditional options for configuring metadata cache have -blocks-storage.bucket-store.metadata-cache.* prefix. By configuring TTL to zero or negative value, caching of given item type is disabled.\nThe same cache backend deployment should be shared between store-gateways and queriers.\nStore-gateway HTTP endpoints GET /store-gateway/ring\nDisplays the status of the store-gateways ring, including the tokens owned by each store-gateway and an option to remove (forget) instances from the ring. Store-gateway configuration This section described the store-gateway configuration. For the general Cortex configuration and references to common config blocks, please refer to the configuration documentation.\nstore_gateway_config The store_gateway_config configures the store-gateway service used by the blocks storage.\nstore_gateway: # Shard blocks across multiple store gateway instances. This option needs be # set both on the store-gateway and querier when running in microservices # mode. # CLI flag: -store-gateway.sharding-enabled [sharding_enabled: \u003cboolean\u003e | default = false] # The hash ring configuration. This option is required only if blocks sharding # is enabled. sharding_ring: # The key-value store used to share the hash ring across multiple instances. # This option needs be set both on the store-gateway and querier when # running in microservices mode. kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -store-gateway.sharding-ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -store-gateway.sharding-ring.prefix [prefix: \u003cstring\u003e | default = \"collectors/\"] dynamodb: # Region to access dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -store-gateway.sharding-ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: # store-gateway.sharding-ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: # store-gateway.sharding-ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -store-gateway.sharding-ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -store-gateway.sharding-ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -store-gateway.sharding-ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -store-gateway.sharding-ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -store-gateway.sharding-ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 15s] # The heartbeat timeout after which store gateways are considered unhealthy # within the ring. 0 = never (timeout disabled). This option needs be set # both on the store-gateway and querier when running in microservices mode. # CLI flag: -store-gateway.sharding-ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # The replication factor to use when sharding blocks. This option needs be # set both on the store-gateway and querier when running in microservices # mode. # CLI flag: -store-gateway.sharding-ring.replication-factor [replication_factor: \u003cint\u003e | default = 3] # File path where tokens are stored. If empty, tokens are not stored at # shutdown and restored at startup. # CLI flag: -store-gateway.sharding-ring.tokens-file-path [tokens_file_path: \u003cstring\u003e | default = \"\"] # True to enable zone-awareness and replicate blocks across different # availability zones. # CLI flag: -store-gateway.sharding-ring.zone-awareness-enabled [zone_awareness_enabled: \u003cboolean\u003e | default = false] # True to keep the store gateway instance in the ring when it shuts down. # The instance will then be auto-forgotten from the ring after # 10*heartbeat_timeout. # CLI flag: -store-gateway.sharding-ring.keep-instance-in-the-ring-on-shutdown [keep_instance_in_the_ring_on_shutdown: \u003cboolean\u003e | default = false] # Minimum time to wait for ring stability at startup. 0 to disable. # CLI flag: -store-gateway.sharding-ring.wait-stability-min-duration [wait_stability_min_duration: \u003cduration\u003e | default = 1m] # Maximum time to wait for ring stability at startup. If the store-gateway # ring keeps changing after this period of time, the store-gateway will # start anyway. # CLI flag: -store-gateway.sharding-ring.wait-stability-max-duration [wait_stability_max_duration: \u003cduration\u003e | default = 5m] # Timeout for waiting on store-gateway to become desired state in the ring. # CLI flag: -store-gateway.sharding-ring.wait-instance-state-timeout [wait_instance_state_timeout: \u003cduration\u003e | default = 10m] # The sleep seconds when store-gateway is shutting down. Need to be close to # or larger than KV Store information propagation delay # CLI flag: -store-gateway.sharding-ring.final-sleep [final_sleep: \u003cduration\u003e | default = 0s] # Name of network interface to read address from. # CLI flag: -store-gateway.sharding-ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # The availability zone where this instance is running. Required if # zone-awareness is enabled. # CLI flag: -store-gateway.sharding-ring.instance-availability-zone [instance_availability_zone: \u003cstring\u003e | default = \"\"] # The sharding strategy to use. Supported values are: default, # shuffle-sharding. # CLI flag: -store-gateway.sharding-strategy [sharding_strategy: \u003cstring\u003e | default = \"default\"] # Comma separated list of tenants whose store metrics this storegateway can # process. If specified, only these tenants will be handled by storegateway, # otherwise this storegateway will be enabled for all the tenants in the # store-gateway cluster. # CLI flag: -store-gateway.enabled-tenants [enabled_tenants: \u003cstring\u003e | default = \"\"] # Comma separated list of tenants whose store metrics this storegateway cannot # process. If specified, a storegateway that would normally pick the specified # tenant(s) for processing will ignore them instead. # CLI flag: -store-gateway.disabled-tenants [disabled_tenants: \u003cstring\u003e | default = \"\"] hedged_request: # If true, hedged requests are applied to object store calls. It can help # with reducing tail latency. # CLI flag: -store-gateway.hedged-request.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of hedged requests allowed for each initial request. A high # number can reduce latency but increase internal calls. # CLI flag: -store-gateway.hedged-request.max-requests [max_requests: \u003cint\u003e | default = 3] # It is used to calculate a latency threshold to trigger hedged requests. # For example, additional requests are triggered when the initial request # response time exceeds the 90th percentile. # CLI flag: -store-gateway.hedged-request.quantile [quantile: \u003cfloat\u003e | default = 0.9] blocks_storage_config The blocks_storage_config configures the blocks storage.\nblocks_storage: # Backend storage to use. Supported backends are: s3, gcs, azure, swift, # filesystem. # CLI flag: -blocks-storage.backend [backend: \u003cstring\u003e | default = \"s3\"] s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an # S3-compatible service in hostname:port format. # CLI flag: -blocks-storage.s3.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # S3 region. If unset, the client will issue a S3 GetBucketLocation API call # to autodetect it. # CLI flag: -blocks-storage.s3.region [region: \u003cstring\u003e | default = \"\"] # S3 bucket name # CLI flag: -blocks-storage.s3.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # S3 secret access key # CLI flag: -blocks-storage.s3.secret-access-key [secret_access_key: \u003cstring\u003e | default = \"\"] # S3 access key ID # CLI flag: -blocks-storage.s3.access-key-id [access_key_id: \u003cstring\u003e | default = \"\"] # If enabled, use http:// for the S3 endpoint instead of https://. This # could be useful in local dev/test environments while using an # S3-compatible backend storage, like Minio. # CLI flag: -blocks-storage.s3.insecure [insecure: \u003cboolean\u003e | default = false] # The signature version to use for authenticating against S3. Supported # values are: v4, v2. # CLI flag: -blocks-storage.s3.signature-version [signature_version: \u003cstring\u003e | default = \"v4\"] # The s3 bucket lookup style. Supported values are: auto, virtual-hosted, # path. # CLI flag: -blocks-storage.s3.bucket-lookup-type [bucket_lookup_type: \u003cstring\u003e | default = \"auto\"] # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum # algorithm to verify the provided digest. If false, use CRC32C algorithm # instead. # CLI flag: -blocks-storage.s3.send-content-md5 [send_content_md5: \u003cboolean\u003e | default = true] # The list api version. Supported values are: v1, v2, and ''. # CLI flag: -blocks-storage.s3.list-objects-version [list_objects_version: \u003cstring\u003e | default = \"\"] # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: blocks-storage [sse: \u003cs3_sse_config\u003e] http: # The time an idle connection will remain idle before closing. # CLI flag: -blocks-storage.s3.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -blocks-storage.s3.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -blocks-storage.s3.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -blocks-storage.s3.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully # writing the request headers if the request has an Expect header. 0 to # send the request body immediately. # CLI flag: -blocks-storage.s3.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 # means no limit. # CLI flag: -blocks-storage.s3.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, # a built-in default value is used. # CLI flag: -blocks-storage.s3.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -blocks-storage.s3.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] gcs: # GCS bucket name # CLI flag: -blocks-storage.gcs.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # JSON representing either a Google Developers Console # client_credentials.json file or a Google Developers service account key # file. If empty, fallback to Google default logic. # CLI flag: -blocks-storage.gcs.service-account [service_account: \u003cstring\u003e | default = \"\"] azure: # Azure storage account name # CLI flag: -blocks-storage.azure.account-name [account_name: \u003cstring\u003e | default = \"\"] # Azure storage account key # CLI flag: -blocks-storage.azure.account-key [account_key: \u003cstring\u003e | default = \"\"] # The values of `account-name` and `endpoint-suffix` values will not be # ignored if `connection-string` is set. Use this method over `account-key` # if you need to authenticate via a SAS token or if you use the Azurite # emulator. # CLI flag: -blocks-storage.azure.connection-string [connection_string: \u003cstring\u003e | default = \"\"] # Azure storage container name # CLI flag: -blocks-storage.azure.container-name [container_name: \u003cstring\u003e | default = \"\"] # Azure storage endpoint suffix without schema. The account name will be # prefixed to this value to create the FQDN # CLI flag: -blocks-storage.azure.endpoint-suffix [endpoint_suffix: \u003cstring\u003e | default = \"\"] # Number of retries for recoverable errors # CLI flag: -blocks-storage.azure.max-retries [max_retries: \u003cint\u003e | default = 20] # Deprecated: Azure storage MSI resource. It will be set automatically by # Azure SDK. # CLI flag: -blocks-storage.azure.msi-resource [msi_resource: \u003cstring\u003e | default = \"\"] # Azure storage MSI resource managed identity client Id. If not supplied # default Azure credential will be used. Set it to empty if you need to # authenticate via Azure Workload Identity. # CLI flag: -blocks-storage.azure.user-assigned-id [user_assigned_id: \u003cstring\u003e | default = \"\"] http: # The time an idle connection will remain idle before closing. # CLI flag: -blocks-storage.azure.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -blocks-storage.azure.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -blocks-storage.azure.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -blocks-storage.azure.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully # writing the request headers if the request has an Expect header. 0 to # send the request body immediately. # CLI flag: -blocks-storage.azure.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 # means no limit. # CLI flag: -blocks-storage.azure.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, # a built-in default value is used. # CLI flag: -blocks-storage.azure.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -blocks-storage.azure.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] swift: # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -blocks-storage.swift.auth-version [auth_version: \u003cint\u003e | default = 0] # OpenStack Swift authentication URL # CLI flag: -blocks-storage.swift.auth-url [auth_url: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential ID. # CLI flag: -blocks-storage.swift.application-credential-id [application_credential_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential name. # CLI flag: -blocks-storage.swift.application-credential-name [application_credential_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential secret. # CLI flag: -blocks-storage.swift.application-credential-secret [application_credential_secret: \u003cstring\u003e | default = \"\"] # OpenStack Swift username. # CLI flag: -blocks-storage.swift.username [username: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -blocks-storage.swift.user-domain-name [user_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -blocks-storage.swift.user-domain-id [user_domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user ID. # CLI flag: -blocks-storage.swift.user-id [user_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift API key. # CLI flag: -blocks-storage.swift.password [password: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -blocks-storage.swift.domain-id [domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -blocks-storage.swift.domain-name [domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift project ID (v2,v3 auth only). # CLI flag: -blocks-storage.swift.project-id [project_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift project name (v2,v3 auth only). # CLI flag: -blocks-storage.swift.project-name [project_name: \u003cstring\u003e | default = \"\"] # ID of the OpenStack Swift project's domain (v3 auth only), only needed if # it differs the from user domain. # CLI flag: -blocks-storage.swift.project-domain-id [project_domain_id: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift project's domain (v3 auth only), only needed # if it differs from the user domain. # CLI flag: -blocks-storage.swift.project-domain-name [project_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift Region to use (v2,v3 auth only). # CLI flag: -blocks-storage.swift.region-name [region_name: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift container to put chunks in. # CLI flag: -blocks-storage.swift.container-name [container_name: \u003cstring\u003e | default = \"\"] # Max retries on requests error. # CLI flag: -blocks-storage.swift.max-retries [max_retries: \u003cint\u003e | default = 3] # Time after which a connection attempt is aborted. # CLI flag: -blocks-storage.swift.connect-timeout [connect_timeout: \u003cduration\u003e | default = 10s] # Time after which an idle request is aborted. The timeout watchdog is reset # each time some data is received, so the timeout triggers after X time no # data is received on a request. # CLI flag: -blocks-storage.swift.request-timeout [request_timeout: \u003cduration\u003e | default = 5s] filesystem: # Local filesystem storage directory. # CLI flag: -blocks-storage.filesystem.dir [dir: \u003cstring\u003e | default = \"\"] # This configures how the querier and store-gateway discover and synchronize # blocks stored in the bucket. bucket_store: # Directory to store synchronized TSDB index headers. # CLI flag: -blocks-storage.bucket-store.sync-dir [sync_dir: \u003cstring\u003e | default = \"tsdb-sync\"] # How frequently to scan the bucket, or to refresh the bucket index (if # enabled), in order to look for changes (new blocks shipped by ingesters # and blocks deleted by retention or compaction). # CLI flag: -blocks-storage.bucket-store.sync-interval [sync_interval: \u003cduration\u003e | default = 15m] # Max number of concurrent queries to execute against the long-term storage. # The limit is shared across all tenants. # CLI flag: -blocks-storage.bucket-store.max-concurrent [max_concurrent: \u003cint\u003e | default = 100] # Max number of inflight queries to execute against the long-term storage. # The limit is shared across all tenants. 0 to disable. # CLI flag: -blocks-storage.bucket-store.max-inflight-requests [max_inflight_requests: \u003cint\u003e | default = 0] # Maximum number of concurrent tenants syncing blocks. # CLI flag: -blocks-storage.bucket-store.tenant-sync-concurrency [tenant_sync_concurrency: \u003cint\u003e | default = 10] # Maximum number of concurrent blocks syncing per tenant. # CLI flag: -blocks-storage.bucket-store.block-sync-concurrency [block_sync_concurrency: \u003cint\u003e | default = 20] # Number of Go routines to use when syncing block meta files from object # storage per tenant. # CLI flag: -blocks-storage.bucket-store.meta-sync-concurrency [meta_sync_concurrency: \u003cint\u003e | default = 20] # Minimum age of a block before it's being read. Set it to safe value (e.g # 30m) if your object storage is eventually consistent. GCS and S3 are # (roughly) strongly consistent. # CLI flag: -blocks-storage.bucket-store.consistency-delay [consistency_delay: \u003cduration\u003e | default = 0s] index_cache: # The index cache backend type. Multiple cache backend can be provided as # a comma-separated ordered list to enable the implementation of a cache # hierarchy. Supported values: inmemory, memcached, redis. # CLI flag: -blocks-storage.bucket-store.index-cache.backend [backend: \u003cstring\u003e | default = \"inmemory\"] inmemory: # Maximum size in bytes of in-memory index cache used to speed up blocks # index lookups (shared between all tenants). # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes [max_size_bytes: \u003cint\u003e | default = 1073741824] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items [enabled_items: \u003clist of string\u003e | default = []] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV # query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup # made after that). # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. # If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should # run. If more keys are specified, internally keys are split into # multiple batches and fetched concurrently, honoring the max # concurrency. If set to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items [enabled_items: \u003clist of string\u003e | default = []] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.index-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching # is when data is stored in memory instead of fetching data each time. # See https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items [enabled_items: \u003clist of string\u003e | default = []] multilevel: # The maximum number of concurrent asynchronous operations can occur # when backfilling cache items. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of items to backfill per asynchronous operation. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-backfill-items [max_backfill_items: \u003cint\u003e | default = 10000] chunks_cache: # The chunks cache backend type. Single or Multiple cache backend can be # provided. Supported values in single cache: memcached, redis, inmemory, # and '' (disable). Supported values in multi level cache: a # comma-separated list of (inmemory, memcached, redis) # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend [backend: \u003cstring\u003e | default = \"\"] inmemory: # Maximum size in bytes of in-memory chunk cache used to speed up chunk # lookups (shared between all tenants). # CLI flag: -blocks-storage.bucket-store.chunks-cache.inmemory.max-size-bytes [max_size_bytes: \u003cint\u003e | default = 1073741824] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV # query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup # made after that). # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. # If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should # run. If more keys are specified, internally keys are split into # multiple batches and fetched concurrently, honoring the max # concurrency. If set to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching # is when data is stored in memory instead of fetching data each time. # See https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] multilevel: # The maximum number of concurrent asynchronous operations can occur # when backfilling cache items. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of items to backfill per asynchronous operation. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-backfill-items [max_backfill_items: \u003cint\u003e | default = 10000] # Size of each subrange that bucket object is split into for better # caching. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-size [subrange_size: \u003cint\u003e | default = 16000] # Maximum number of sub-GetRange requests that a single GetRange request # can be split into when fetching chunks. Zero or negative value = # unlimited number of sub-requests. # CLI flag: -blocks-storage.bucket-store.chunks-cache.max-get-range-requests [max_get_range_requests: \u003cint\u003e | default = 3] # TTL for caching object attributes for chunks. # CLI flag: -blocks-storage.bucket-store.chunks-cache.attributes-ttl [attributes_ttl: \u003cduration\u003e | default = 168h] # TTL for caching individual chunks subranges. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-ttl [subrange_ttl: \u003cduration\u003e | default = 24h] metadata_cache: # Backend for metadata cache, if not empty. Supported values: memcached, # redis, and '' (disable). # CLI flag: -blocks-storage.bucket-store.metadata-cache.backend [backend: \u003cstring\u003e | default = \"\"] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV # query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup # made after that). # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. # If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should # run. If more keys are specified, internally keys are split into # multiple batches and fetched concurrently, honoring the max # concurrency. If set to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching # is when data is stored in memory instead of fetching data each time. # See https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # How long to cache list of tenants in the bucket. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl [tenants_list_ttl: \u003cduration\u003e | default = 15m] # How long to cache list of blocks for each tenant. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenant-blocks-list-ttl [tenant_blocks_list_ttl: \u003cduration\u003e | default = 5m] # How long to cache list of chunks for a block. # CLI flag: -blocks-storage.bucket-store.metadata-cache.chunks-list-ttl [chunks_list_ttl: \u003cduration\u003e | default = 24h] # How long to cache information that block metafile exists. Also used for # user deletion mark file. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-exists-ttl [metafile_exists_ttl: \u003cduration\u003e | default = 2h] # How long to cache information that block metafile doesn't exist. Also # used for user deletion mark file. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-doesnt-exist-ttl [metafile_doesnt_exist_ttl: \u003cduration\u003e | default = 5m] # How long to cache content of the metafile. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-content-ttl [metafile_content_ttl: \u003cduration\u003e | default = 24h] # Maximum size of metafile content to cache in bytes. Caching will be # skipped if the content exceeds this size. This is useful to avoid # network round trip for large content if the configured caching backend # has an hard limit on cached items size (in this case, you should set # this limit to the same limit in the caching backend). # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-max-size-bytes [metafile_max_size_bytes: \u003cint\u003e | default = 1048576] # How long to cache attributes of the block metafile. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-attributes-ttl [metafile_attributes_ttl: \u003cduration\u003e | default = 168h] # How long to cache attributes of the block index. # CLI flag: -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl [block_index_attributes_ttl: \u003cduration\u003e | default = 168h] # How long to cache content of the bucket index. # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl [bucket_index_content_ttl: \u003cduration\u003e | default = 5m] # Maximum size of bucket index content to cache in bytes. Caching will be # skipped if the content exceeds this size. This is useful to avoid # network round trip for large content if the configured caching backend # has an hard limit on cached items size (in this case, you should set # this limit to the same limit in the caching backend). # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-max-size-bytes [bucket_index_max_size_bytes: \u003cint\u003e | default = 1048576] # Duration after which the blocks marked for deletion will be filtered out # while fetching blocks. The idea of ignore-deletion-marks-delay is to # ignore blocks that are marked for deletion with some delay. This ensures # store can still serve blocks that are meant to be deleted but do not have # a replacement yet. Default is 6h, half of the default value for # -compactor.deletion-delay. # CLI flag: -blocks-storage.bucket-store.ignore-deletion-marks-delay [ignore_deletion_mark_delay: \u003cduration\u003e | default = 6h] # The blocks created since `now() - ignore_blocks_within` will not be # synced. This should be used together with `-querier.query-store-after` to # filter out the blocks that are too new to be queried. A reasonable value # for this flag would be `-querier.query-store-after - # blocks-storage.bucket-store.bucket-index.max-stale-period` to give some # buffer. 0 to disable. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: \u003cduration\u003e | default = 0s] bucket_index: # True to enable querier and store-gateway to discover blocks in the # storage via bucket index instead of bucket scanning. # CLI flag: -blocks-storage.bucket-store.bucket-index.enabled [enabled: \u003cboolean\u003e | default = false] # How frequently a bucket index, which previously failed to load, should # be tried to load again. This option is used only by querier. # CLI flag: -blocks-storage.bucket-store.bucket-index.update-on-error-interval [update_on_error_interval: \u003cduration\u003e | default = 1m] # How long a unused bucket index should be cached. Once this timeout # expires, the unused bucket index is removed from the in-memory cache. # This option is used only by querier. # CLI flag: -blocks-storage.bucket-store.bucket-index.idle-timeout [idle_timeout: \u003cduration\u003e | default = 1h] # The maximum allowed age of a bucket index (last updated) before queries # start failing because the bucket index is too old. The bucket index is # periodically updated by the compactor, while this check is enforced in # the querier (at query time). # CLI flag: -blocks-storage.bucket-store.bucket-index.max-stale-period [max_stale_period: \u003cduration\u003e | default = 1h] # One of concurrent, recursive, bucket_index. When set to concurrent, stores # will concurrently issue one call per directory to discover active blocks # in the bucket. The recursive strategy iterates through all objects in the # bucket, recursively traversing into each directory. This avoids N+1 calls # at the expense of having slower bucket iterations. bucket_index strategy # can be used in Compactor only and utilizes the existing bucket index to # fetch block IDs to sync. This avoids iterating the bucket but can be # impacted by delays of cleaner creating bucket index. # CLI flag: -blocks-storage.bucket-store.block-discovery-strategy [block_discovery_strategy: \u003cstring\u003e | default = \"concurrent\"] # Max size - in bytes - of a chunks pool, used to reduce memory allocations. # The pool is shared across all tenants. 0 to disable the limit. # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes [max_chunk_pool_bytes: \u003cint\u003e | default = 2147483648] # If enabled, store-gateway will lazily memory-map an index-header only once # required by a query. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-enabled [index_header_lazy_loading_enabled: \u003cboolean\u003e | default = false] # If index-header lazy loading is enabled and this setting is \u003e 0, the # store-gateway will release memory-mapped index-headers after 'idle # timeout' inactivity. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout [index_header_lazy_loading_idle_timeout: \u003cduration\u003e | default = 20m] # If true, Store Gateway will estimate postings size and try to lazily # expand postings if it downloads less data than expanding all postings. # CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled [lazy_expanded_postings_enabled: \u003cboolean\u003e | default = false] # Controls how many series to fetch per batch in Store Gateway. Default # value is 10000. # CLI flag: -blocks-storage.bucket-store.series-batch-size [series_batch_size: \u003cint\u003e | default = 10000] token_bucket_bytes_limiter: # Token bucket bytes limiter mode. Supported values are: disabled, dryrun, # enabled # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.mode [mode: \u003cstring\u003e | default = \"disabled\"] # Instance token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.instance-token-bucket-size [instance_token_bucket_size: \u003cint\u003e | default = 859832320] # User token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.user-token-bucket-size [user_token_bucket_size: \u003cint\u003e | default = 644874240] # Request token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.request-token-bucket-size [request_token_bucket_size: \u003cint\u003e | default = 4194304] tsdb: # Local directory to store TSDBs in the ingesters. # CLI flag: -blocks-storage.tsdb.dir [dir: \u003cstring\u003e | default = \"tsdb\"] # TSDB blocks range period. # CLI flag: -blocks-storage.tsdb.block-ranges-period [block_ranges_period: \u003clist of duration\u003e | default = 2h0m0s] # TSDB blocks retention in the ingester before a block is removed. This # should be larger than the block_ranges_period and large enough to give # store-gateways and queriers enough time to discover newly uploaded blocks. # CLI flag: -blocks-storage.tsdb.retention-period [retention_period: \u003cduration\u003e | default = 6h] # How frequently the TSDB blocks are scanned and new ones are shipped to the # storage. 0 means shipping is disabled. # CLI flag: -blocks-storage.tsdb.ship-interval [ship_interval: \u003cduration\u003e | default = 1m] # Maximum number of tenants concurrently shipping blocks to the storage. # CLI flag: -blocks-storage.tsdb.ship-concurrency [ship_concurrency: \u003cint\u003e | default = 10] # How frequently does Cortex try to compact TSDB head. Block is only created # if data covers smallest block range. Must be greater than 0 and max 30 # minutes. Note that up to 50% jitter is added to the value for the first # compaction to avoid ingesters compacting concurrently. # CLI flag: -blocks-storage.tsdb.head-compaction-interval [head_compaction_interval: \u003cduration\u003e | default = 1m] # Maximum number of tenants concurrently compacting TSDB head into a new # block # CLI flag: -blocks-storage.tsdb.head-compaction-concurrency [head_compaction_concurrency: \u003cint\u003e | default = 5] # If TSDB head is idle for this duration, it is compacted. Note that up to # 25% jitter is added to the value to avoid ingesters compacting # concurrently. 0 means disabled. # CLI flag: -blocks-storage.tsdb.head-compaction-idle-timeout [head_compaction_idle_timeout: \u003cduration\u003e | default = 1h] # The write buffer size used by the head chunks mapper. Lower values reduce # memory utilisation on clusters with a large number of tenants at the cost # of increased disk I/O operations. # CLI flag: -blocks-storage.tsdb.head-chunks-write-buffer-size-bytes [head_chunks_write_buffer_size_bytes: \u003cint\u003e | default = 4194304] # The number of shards of series to use in TSDB (must be a power of 2). # Reducing this will decrease memory footprint, but can negatively impact # performance. # CLI flag: -blocks-storage.tsdb.stripe-size [stripe_size: \u003cint\u003e | default = 16384] # Deprecated (use blocks-storage.tsdb.wal-compression-type instead): True to # enable TSDB WAL compression. # CLI flag: -blocks-storage.tsdb.wal-compression-enabled [wal_compression_enabled: \u003cboolean\u003e | default = false] # TSDB WAL type. Supported values are: 'snappy', 'zstd' and '' (disable # compression) # CLI flag: -blocks-storage.tsdb.wal-compression-type [wal_compression_type: \u003cstring\u003e | default = \"\"] # TSDB WAL segments files max size (bytes). # CLI flag: -blocks-storage.tsdb.wal-segment-size-bytes [wal_segment_size_bytes: \u003cint\u003e | default = 134217728] # True to flush blocks to storage on shutdown. If false, incomplete blocks # will be reused after restart. # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown [flush_blocks_on_shutdown: \u003cboolean\u003e | default = false] # If TSDB has not received any data for this duration, and all blocks from # TSDB have been shipped, TSDB is closed and deleted from local disk. If set # to positive value, this value should be equal or higher than # -querier.query-ingesters-within flag to make sure that TSDB is not closed # prematurely, which could cause partial query results. 0 or negative value # disables closing of idle TSDB. # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout [close_idle_tsdb_timeout: \u003cduration\u003e | default = 0s] # The size of the in-memory queue used before flushing chunks to the disk. # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: \u003cint\u003e | default = 0] # limit the number of concurrently opening TSDB's on startup # CLI flag: -blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup [max_tsdb_opening_concurrency_on_startup: \u003cint\u003e | default = 10] # Deprecated, use maxExemplars in limits instead. If the MaxExemplars value # in limits is set to zero, cortex will fallback on this value. This setting # enables support for exemplars in TSDB and sets the maximum number that # will be stored. 0 or less means disabled. # CLI flag: -blocks-storage.tsdb.max-exemplars [max_exemplars: \u003cint\u003e | default = 0] # True to enable snapshotting of in-memory TSDB data on disk when shutting # down. # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown [memory_snapshot_on_shutdown: \u003cboolean\u003e | default = false] # [EXPERIMENTAL] Configures the maximum number of samples per chunk that can # be out-of-order. # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max [out_of_order_cap_max: \u003cint\u003e | default = 32] # [EXPERIMENTAL] True to enable native histogram. # CLI flag: -blocks-storage.tsdb.enable-native-histograms [enable_native_histograms: \u003cboolean\u003e | default = false] # [EXPERIMENTAL] If enabled, ingesters will cache expanded postings when # querying blocks. Caching can be configured separately for the head and # compacted blocks. expanded_postings_cache: # If enabled, ingesters will cache expanded postings for the head block. # Only queries with with an equal matcher for metric __name__ are cached. head: # Whether the postings cache is enabled or not # CLI flag: -blocks-storage.expanded_postings_cache.head.enabled [enabled: \u003cboolean\u003e | default = false] # Max bytes for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.head.max-bytes [max_bytes: \u003cint\u003e | default = 10485760] # TTL for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.head.ttl [ttl: \u003cduration\u003e | default = 10m] # If enabled, ingesters will cache expanded postings for the compacted # blocks. The cache is shared between all blocks. blocks: # Whether the postings cache is enabled or not # CLI flag: -blocks-storage.expanded_postings_cache.block.enabled [enabled: \u003cboolean\u003e | default = false] # Max bytes for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.block.max-bytes [max_bytes: \u003cint\u003e | default = 10485760] # TTL for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.block.ttl [ttl: \u003cduration\u003e | default = 10m] ","categories":"","description":"","excerpt":" The store-gateway is the Cortex service responsible to query series …","ref":"/docs/blocks-storage/store-gateway/","tags":"","title":"Store-gateway"},{"body":" The compactor is an service which is responsible to:\nCompact multiple blocks of a given tenant into a single optimized larger block. This helps to reduce storage costs (deduplication, index size reduction), and increase query speed (querying fewer blocks is faster). Keep the per-tenant bucket index updated. The bucket index is used by queriers, store-gateways and rulers to discover new blocks in the storage. The compactor is stateless.\nHow compaction works The blocks compaction has two main benefits:\nVertically compact blocks uploaded by all ingesters for the same time range Horizontally compact blocks with small time ranges into a single larger block The vertical compaction merges all the blocks of a tenant uploaded by ingesters for the same time range (2 hours ranges by default) into a single block, while de-duplicating samples that are originally written to N blocks as a result of replication. This step reduces number of blocks for single 2 hours time range from #(number of ingesters) to 1 per tenant.\nThe horizontal compaction triggers after the vertical compaction and compacts several blocks with adjacent 2-hour range periods into a single larger block. Even though the total size of block chunks doesn’t change after this compaction, it may still significantly reduce the size of the index and the index-header kept in memory by store-gateways.\nCompactor sharding The compactor optionally supports sharding.\nWhen sharding is enabled, multiple compactor instances can coordinate to split the workload and shard blocks by tenant. All the blocks of a tenant are processed by a single compactor instance at any given time, but compaction for different tenants may simultaneously run on different compactor instances.\nWhenever the pool of compactors increase or decrease (ie. following up a scale up/down), tenants are resharded across the available compactor instances without any manual intervention.\nThe compactor sharding is based on the Cortex hash ring. At startup, a compactor generates random tokens and registers itself to the ring. While running, it periodically scans the storage bucket (every -compactor.compaction-interval) to discover the list of tenants in the storage and compacts blocks for each tenant whose hash matches the token ranges assigned to the instance itself within the ring.\nThis feature can be enabled via -compactor.sharding-enabled=true and requires the backend hash ring to be configured via -compactor.ring.* flags (or their respective YAML config options).\nWaiting for stable ring at startup In the event of a cluster cold start or scale up of 2+ compactor instances at the same time we may end up in a situation where each new compactor instance starts at a slightly different time and thus each one runs the first compaction based on a different state of the ring. This is not a critical condition, but may be inefficient, because multiple compactor replicas may start compacting the same tenant nearly at the same time.\nTo reduce the likelihood this could happen, the compactor waits for a stable ring at startup. A ring is considered stable if no instance is added/removed to the ring for at least -compactor.ring.wait-stability-min-duration. If the ring keep getting changed after -compactor.ring.wait-stability-max-duration, the compactor will stop waiting for a stable ring and will proceed starting up normally.\nTo disable this waiting logic, you can start the compactor with -compactor.ring.wait-stability-min-duration=0.\nSoft and hard blocks deletion When the compactor successfully compacts some source blocks into a larger block, source blocks are deleted from the storage. Blocks deletion is not immediate, but follows a two steps process:\nFirst, a block is marked for deletion (soft delete) Then, once a block is marked for deletion for longer then -compactor.deletion-delay, the block is deleted from the storage (hard delete) The compactor is both responsible to mark blocks for deletion and then hard delete them once the deletion delay expires. The soft deletion is based on a tiny deletion-mark.json file stored within the block location in the bucket which gets looked up both by queriers and store-gateways.\nThis soft deletion mechanism is used to give enough time to queriers and store-gateways to discover the new compacted blocks before the old source blocks are deleted. If source blocks would be immediately hard deleted by the compactor, some queries involving the compacted blocks may fail until the queriers and store-gateways haven’t rescanned the bucket and found both deleted source blocks and the new compacted ones.\nCompactor disk utilization The compactor needs to download source blocks from the bucket to the local disk, and store the compacted block to the local disk before uploading it to the bucket. Depending on the largest tenants in your cluster and the configured -compactor.block-ranges, the compactor may need a lot of disk space.\nAssuming max_compaction_range_blocks_size is the total size of blocks for the largest tenant (you can measure it inspecting the bucket) and the longest -compactor.block-ranges period, the formula to estimate the minimum disk space required is:\nmin_disk_space_required = compactor.compaction-concurrency * max_compaction_range_blocks_size * 2 Alternatively, assuming the largest -compactor.block-ranges is 24h (default), you could consider 150GB of disk space every 10M active series owned by the largest tenant. For example, if your largest tenant has 30M active series and -compactor.compaction-concurrency=1 we would recommend having a disk with at least 450GB available.\nCompactor HTTP endpoints GET /compactor/ring\nDisplays the status of the compactors ring, including the tokens owned by each compactor and an option to remove (forget) instances from the ring. Compactor configuration This section described the compactor configuration. For the general Cortex configuration and references to common config blocks, please refer to the configuration documentation.\ncompactor_config The compactor_config configures the compactor for the blocks storage.\ncompactor: # List of compaction time ranges. # CLI flag: -compactor.block-ranges [block_ranges: \u003clist of duration\u003e | default = 2h0m0s,12h0m0s,24h0m0s] # Number of Go routines to use when syncing block index and chunks files from # the long term storage. # CLI flag: -compactor.block-sync-concurrency [block_sync_concurrency: \u003cint\u003e | default = 20] # Number of Go routines to use when syncing block meta files from the long # term storage. # CLI flag: -compactor.meta-sync-concurrency [meta_sync_concurrency: \u003cint\u003e | default = 20] # Minimum age of fresh (non-compacted) blocks before they are being processed. # Malformed blocks older than the maximum of consistency-delay and 48h0m0s # will be removed. # CLI flag: -compactor.consistency-delay [consistency_delay: \u003cduration\u003e | default = 0s] # Data directory in which to cache blocks and process compactions # CLI flag: -compactor.data-dir [data_dir: \u003cstring\u003e | default = \"./data\"] # The frequency at which the compaction runs # CLI flag: -compactor.compaction-interval [compaction_interval: \u003cduration\u003e | default = 1h] # How many times to retry a failed compaction within a single compaction run. # CLI flag: -compactor.compaction-retries [compaction_retries: \u003cint\u003e | default = 3] # Max number of concurrent compactions running. # CLI flag: -compactor.compaction-concurrency [compaction_concurrency: \u003cint\u003e | default = 1] # How frequently compactor should run blocks cleanup and maintenance, as well # as update the bucket index. # CLI flag: -compactor.cleanup-interval [cleanup_interval: \u003cduration\u003e | default = 15m] # Max number of tenants for which blocks cleanup and maintenance should run # concurrently. # CLI flag: -compactor.cleanup-concurrency [cleanup_concurrency: \u003cint\u003e | default = 20] # Time before a block marked for deletion is deleted from bucket. If not 0, # blocks will be marked for deletion and compactor component will permanently # delete blocks marked for deletion from the bucket. If 0, blocks will be # deleted straight away. Note that deleting blocks immediately can cause query # failures. # CLI flag: -compactor.deletion-delay [deletion_delay: \u003cduration\u003e | default = 12h] # For tenants marked for deletion, this is time between deleting of last # block, and doing final cleanup (marker files, debug files) of the tenant. # CLI flag: -compactor.tenant-cleanup-delay [tenant_cleanup_delay: \u003cduration\u003e | default = 6h] # When enabled, mark blocks containing index with out-of-order chunks for no # compact instead of halting the compaction. # CLI flag: -compactor.skip-blocks-with-out-of-order-chunks-enabled [skip_blocks_with_out_of_order_chunks_enabled: \u003cboolean\u003e | default = false] # Number of goroutines to use when fetching/uploading block files from object # storage. # CLI flag: -compactor.block-files-concurrency [block_files_concurrency: \u003cint\u003e | default = 10] # Number of goroutines to use when fetching blocks from object storage when # compacting. # CLI flag: -compactor.blocks-fetch-concurrency [blocks_fetch_concurrency: \u003cint\u003e | default = 3] # When enabled, at compactor startup the bucket will be scanned and all found # deletion marks inside the block location will be copied to the markers # global location too. This option can (and should) be safely disabled as soon # as the compactor has successfully run at least once. # CLI flag: -compactor.block-deletion-marks-migration-enabled [block_deletion_marks_migration_enabled: \u003cboolean\u003e | default = false] # Comma separated list of tenants that can be compacted. If specified, only # these tenants will be compacted by compactor, otherwise all tenants can be # compacted. Subject to sharding. # CLI flag: -compactor.enabled-tenants [enabled_tenants: \u003cstring\u003e | default = \"\"] # Comma separated list of tenants that cannot be compacted by this compactor. # If specified, and compactor would normally pick given tenant for compaction # (via -compactor.enabled-tenants or sharding), it will be ignored instead. # CLI flag: -compactor.disabled-tenants [disabled_tenants: \u003cstring\u003e | default = \"\"] # Shard tenants across multiple compactor instances. Sharding is required if # you run multiple compactor instances, in order to coordinate compactions and # avoid race conditions leading to the same tenant blocks simultaneously # compacted by different instances. # CLI flag: -compactor.sharding-enabled [sharding_enabled: \u003cboolean\u003e | default = false] # The sharding strategy to use. Supported values are: default, # shuffle-sharding. # CLI flag: -compactor.sharding-strategy [sharding_strategy: \u003cstring\u003e | default = \"default\"] sharding_ring: kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -compactor.ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -compactor.ring.prefix [prefix: \u003cstring\u003e | default = \"collectors/\"] dynamodb: # Region to access dynamodb. # CLI flag: -compactor.ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -compactor.ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -compactor.ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -compactor.ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -compactor.ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: compactor.ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: compactor.ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -compactor.ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -compactor.ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -compactor.ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -compactor.ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -compactor.ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 5s] # The heartbeat timeout after which compactors are considered unhealthy # within the ring. 0 = never (timeout disabled). # CLI flag: -compactor.ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # Minimum time to wait for ring stability at startup. 0 to disable. # CLI flag: -compactor.ring.wait-stability-min-duration [wait_stability_min_duration: \u003cduration\u003e | default = 1m] # Maximum time to wait for ring stability at startup. If the compactor ring # keeps changing after this period of time, the compactor will start anyway. # CLI flag: -compactor.ring.wait-stability-max-duration [wait_stability_max_duration: \u003cduration\u003e | default = 5m] # Name of network interface to read address from. # CLI flag: -compactor.ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # File path where tokens are stored. If empty, tokens are not stored at # shutdown and restored at startup. # CLI flag: -compactor.ring.tokens-file-path [tokens_file_path: \u003cstring\u003e | default = \"\"] # Unregister the compactor during shutdown if true. # CLI flag: -compactor.ring.unregister-on-shutdown [unregister_on_shutdown: \u003cboolean\u003e | default = true] # Timeout for waiting on compactor to become ACTIVE in the ring. # CLI flag: -compactor.ring.wait-active-instance-timeout [wait_active_instance_timeout: \u003cduration\u003e | default = 10m] # The compaction strategy to use. Supported values are: default, partitioning. # CLI flag: -compactor.compaction-mode [compaction_mode: \u003cstring\u003e | default = \"default\"] # How long block visit marker file should be considered as expired and able to # be picked up by compactor again. # CLI flag: -compactor.block-visit-marker-timeout [block_visit_marker_timeout: \u003cduration\u003e | default = 5m] # How frequently block visit marker file should be updated duration # compaction. # CLI flag: -compactor.block-visit-marker-file-update-interval [block_visit_marker_file_update_interval: \u003cduration\u003e | default = 1m] # How long cleaner visit marker file should be considered as expired and able # to be picked up by cleaner again. The value should be smaller than # -compactor.cleanup-interval # CLI flag: -compactor.cleaner-visit-marker-timeout [cleaner_visit_marker_timeout: \u003cduration\u003e | default = 10m] # How frequently cleaner visit marker file should be updated when cleaning # user. # CLI flag: -compactor.cleaner-visit-marker-file-update-interval [cleaner_visit_marker_file_update_interval: \u003cduration\u003e | default = 5m] # When enabled, index verification will ignore out of order label names. # CLI flag: -compactor.accept-malformed-index [accept_malformed_index: \u003cboolean\u003e | default = false] # When enabled, caching bucket will be used for compactor, except cleaner # service, which serves as the source of truth for block status # CLI flag: -compactor.caching-bucket-enabled [caching_bucket_enabled: \u003cboolean\u003e | default = false] ","categories":"","description":"","excerpt":" The compactor is an service which is responsible to:\nCompact multiple …","ref":"/docs/blocks-storage/compactor/","tags":"","title":"Compactor"},{"body":"Cortex requires a Key-Value (KV) store to store the ring. It can use traditional KV stores like Consul or Etcd, but it can also build its own KV store on top of the memberlist library using a gossip algorithm.\nThis short guide shows how to start Cortex in single-binary mode with a memberlist-based ring. To reduce the number of required dependencies in this guide, it will use blocks storage with no shipping to external stores. The storage engine and external storage configuration are not dependent on the ring configuration.\nSingle-binary, two Cortex instances For simplicity and to get started, we’ll run it as two instances of Cortex on the local computer. We will use prepared configuration files (file 1, file 2), with no external dependencies.\nBuild Cortex first:\n$ go build ./cmd/cortex Run two instances of Cortex, each one with its own dedicated config file:\n$ ./cortex -config.file docs/configuration/single-process-config-blocks-gossip-1.yaml $ ./cortex -config.file docs/configuration/single-process-config-blocks-gossip-2.yaml Download Prometheus and configure it to use our first Cortex instance for remote writes.\nremote_write: - url: http://localhost:9109/api/v1/push After starting Prometheus, it will now start pushing data to Cortex. The Distributor component in Cortex will distribute incoming samples between the two instances.\nTo query that data, you can configure your Grafana instance to use http://localhost:9109/prometheus (first Cortex) as a Prometheus data source.\nHow it works The two instances we started earlier should be able to find each other via memberlist configuration (already present in the config files):\nmemberlist: # defaults to hostname node_name: \"Ingester 1\" bind_port: 7946 join_members: - localhost:7947 abort_if_cluster_join_fails: false This tells memberlist to listen on port 7946, and connect to localhost:7947, which is the second instance. Port numbers are reversed in the second configuration file. We also need to configure node_name and also ingester ID (ingester.lifecycler.id field), because it defaults to hostname, but we are running both Cortex instances on the same host.\nTo make sure that both ingesters generate unique tokens, we configure join_after and observe_period to 10 seconds. The first option tells Cortex to wait 10 seconds before joining the ring. This option is normally used to tell Cortex ingester how long to wait for a potential token and data transfer from leaving ingester, but we also use it here to increase the chance of finding other gossip peers. When Cortex joins the ring, it generates tokens and writes them to the ring. If multiple Cortex instances do this at the same time, they can generate conflicting tokens. This can be a problem when using a gossiped ring (instances may simply not see each other yet), so we use observe_period to watch the ring for token conflicts. If conflict is detected, new tokens are generated instead of conflicting tokens, and the observe period is restarted. If no conflict is detected within the observe period, the ingester switches to the ACTIVE state.\nWe are able to observe ring state on http://localhost:9109/ring and http://localhost:9209/ring. The two instances may see slightly different views (eg. different timestamps), but they should converge to a common state soon, with both instances being ACTIVE and ready to receive samples.\nHow to add another instance? To add another Cortex to the small cluster, copy docs/configuration/single-process-config-blocks-gossip-1.yaml to a new file, and make the following modifications. We assume that the third Cortex will run on the same machine again, so we change the node name and ingester ID as well. Here is the annotated diff:\n... server: + # These ports need to be unique. - http_listen_port: 9109 - grpc_listen_port: 9195 + http_listen_port: 9309 + grpc_listen_port: 9395 ... ingester: lifecycler: # Defaults to hostname, but we run both ingesters in this demonstration on the same machine. - id: \"Ingester 1\" + id: \"Ingester 3\" ... memberlist: # defaults to hostname - node_name: \"Ingester 1\" + node_name: \"Ingester 3\" # bind_port needs to be unique - bind_port: 7946 + bind_port: 7948 ... +# Directory names in the `blocks_storage` \u003e `tsdb` config ending with `...1` to end with `...3`. This is to avoid different instances +# writing in-progress data to the same directories. blocks_storage: tsdb: - dir: /tmp/cortex/tsdb-ing1 + dir: /tmp/cortex/tsdb-ing3 bucket_store: - sync_dir: /tmp/cortex/tsdb-sync-querier1 + sync_dir: /tmp/cortex/tsdb-sync-querier3 ... We don’t need to change or add memberlist.join_members list. This new instance will simply join to the second one (listening on port 7947), and will discover other peers through it. When using Kubernetes, the suggested setup is to have a headless service pointing to all pods that want to be part of the gossip cluster, and then point join_members to this headless service.\nWe also don’t need to change /tmp/cortex/storage directory in the blocks_storage.filesystem.dir field. This is the directory where all ingesters will “upload” finished blocks. This can also be an S3 or GCP storage, but for simplicity, we use the local filesystem in this example.\nAfter these changes, we can start another Cortex instance using the modified configuration file. This instance will join the ring and will start receiving samples after it enters the ACTIVE state.\n","categories":"","description":"","excerpt":"Cortex requires a Key-Value (KV) store to store the ring. It can use …","ref":"/docs/guides/getting-started-with-gossiped-ring/","tags":"","title":"Getting started with gossiped ring"},{"body":"To upgrade the Go’s runtime version:\nUpgrade build image version with golang version as describe here If the minimum support Go’s language version should be upgraded as well:\nUpgrade go version in go.mod ","categories":"","description":"","excerpt":"To upgrade the Go’s runtime version:\nUpgrade build image version with …","ref":"/docs/contributing/how-to-upgrade-golang-version/","tags":"","title":"How to upgrade Golang version"},{"body":"This page shares some tips and things to take in consideration when setting up a production Cortex cluster based on the blocks storage.\nIngester Ensure a high number of max open file descriptors The ingester stores received series into per-tenant TSDB blocks. Both TSDB WAL, head and compacted blocks are composed by a relatively large number of files which gets loaded via mmap. This means that the ingester keeps file descriptors open for TSDB WAL segments, chunk files and compacted blocks which haven’t reached the retention period yet.\nIf your Cortex cluster has many tenants or ingester is running with a long -blocks-storage.tsdb.retention-period, the ingester may hit the file-max ulimit (maximum number of open file descriptions by a process); in such case, we recommend increasing the limit on your system or enabling shuffle sharding.\nThe rule of thumb is that a production system shouldn’t have the file-max ulimit below 65536, but higher values are recommended (eg. 1048576).\nIngester disk space Ingesters create blocks on disk as samples come in, then every 2 hours (configurable) they cut off those blocks and start a new block.\nWe typically configure ingesters to retain these blocks for longer, to allow time to recover if something goes wrong uploading to the long-term store and to reduce work in queriers - more detail here.\nIf you configure ingesters with -blocks-storage.tsdb.retention-period=24h, a rule of thumb for disk space required is to take the number of timeseries after replication and multiply by 30KB.\nFor example, if you have 20M active series replicated 3 ways, this gives approx 1.7TB. Divide by the number of ingesters and allow some margin for growth, e.g. if you have 20 ingesters then 100GB each should work, or 150GB each to be more comfortable.\nQuerier Ensure caching is enabled The querier relies on caching to reduce the number API calls to the storage bucket. Ensure caching is properly configured and properly scaled.\nEnsure bucket index is enabled The bucket index reduces the number of API calls to the storage bucket and, when enabled, the querier is up and running immediately after the startup (no need to run an initial bucket scan). Ensure bucket index is enabled for the querier.\nAvoid querying non compacted blocks When running Cortex blocks storage cluster at scale, querying non compacted blocks may be inefficient for two reasons:\nNon compacted blocks contain duplicated samples (as effect of the ingested samples replication) Overhead introduced querying many small indexes Because of this, we would suggest to avoid querying non compacted blocks. In order to do it, you should:\nRun the compactor Configure queriers -querier.query-store-after large enough to give compactor enough time to compact newly uploaded blocks (see below) Configure queriers -querier.query-ingesters-within equal to -querier.query-store-after plus 5m (5 minutes is just a delta to query the boundary both from ingesters and queriers) Configure ingesters -blocks-storage.tsdb.retention-period at least as -querier.query-ingesters-within Lower -blocks-storage.bucket-store.ignore-deletion-marks-delay to 1h, otherwise non compacted blocks could be queried anyway, even if their compacted replacement is available How to estimate -querier.query-store-after The -querier.query-store-after should be set to a duration large enough to give compactor enough time to compact newly uploaded blocks, and queriers and store-gateways to discover and sync newly compacted blocks.\nThe following diagram shows all the timings involved in the estimation. This diagram should be used only as a template and you’re expected to tweak the assumptions based on real measurements in your Cortex cluster. In this example, the following assumptions have been done:\nAn ingester takes up to 30 minutes to upload a block to the storage The compactor takes up to 3 hours to compact 2h blocks shipped from all ingesters Querier and store-gateways take up to 15 minutes to discover and load a new compacted block Given these assumptions, in the worst case scenario it would take up to 6h and 45m since when a sample has been ingested until that sample has been appended to a block flushed to the storage and that block has been vertically compacted with all other overlapping 2h blocks shipped from ingesters.\nStore-gateway Ensure caching is enabled The store-gateway heavily relies on caching both to speed up the queries and to reduce the number of API calls to the storage bucket. Ensure caching is properly configured and properly scaled.\nEnsure bucket index is enabled The bucket index reduces the number of API calls to the storage bucket and the startup time of the store-gateway. Ensure bucket index is enabled for the store-gateway.\nEnsure a high number of max open file descriptors The store-gateway stores each block’s index-header on the local disk and loads it via mmap. This means that the store-gateway keeps a file descriptor open for each loaded block. If your Cortex cluster has many blocks in the bucket, the store-gateway may hit the file-max ulimit (maximum number of open file descriptions by a process); in such case, we recommend increasing the limit on your system or running more store-gateway instances with blocks sharding enabled.\nThe rule of thumb is that a production system shouldn’t have the file-max ulimit below 65536, but higher values are recommended (eg. 1048576).\nCompactor Ensure the compactor has enough disk space The compactor generally needs a lot of disk space in order to download source blocks from the bucket and store the compacted block before uploading it to the storage. Please refer to Compactor disk utilization for more information about how to do capacity planning.\nCaching Ensure memcached is properly scaled The rule of thumb to ensure memcached is properly scaled is to make sure evictions happen infrequently. When that’s not the case and they affect query performances, the suggestion is to scale out the memcached cluster adding more nodes or increasing the memory limit of existing ones.\nWe also recommend to run a different memcached cluster for each cache type (metadata, index, chunks). It’s not required, but suggested to not worry about the effect of memory pressure on a cache type against others.\nAlertmanager Ensure Alertmanager networking is hardened If the Alertmanager API is enabled, users with access to Cortex can autonomously configure the Alertmanager, including receiver integrations that allow to issue network requests to the configured URL (eg. webhook). If the Alertmanager network is not hardened, Cortex users may have the ability to issue network requests to any network endpoint including services running in the local network accessible by the Alertmanager itself.\nDespite hardening the system is out of the scope of Cortex, Cortex provides a basic built-in firewall to block connections created by Alertmanager receiver integrations:\n-alertmanager.receivers-firewall-block-cidr-networks -alertmanager.receivers-firewall-block-private-addresses These settings can also be overridden on a per-tenant basis via overrides specified in the runtime config.\n","categories":"","description":"","excerpt":"This page shares some tips and things to take in consideration when …","ref":"/docs/blocks-storage/production-tips/","tags":"","title":"Production tips"},{"body":"Requests mirroring (or shadowing) is a technique you can use to mirror requests from a primary Cortex cluster to a secondary one.\nFor example, requests mirroring can be used when you need to setup a testing Cortex cluster receiving the same series ingested by a primary one without having control over Prometheus remote write config (if you do, then configuring two remote write entries in Prometheus would be the preferred option).\nMirroring with Envoy proxy Envoy proxy can be used to mirror HTTP requests to a secondary upstream cluster. From a network path perspective, you should run Envoy in front of both clusters distributors, letting Envoy to proxy requests to the primary Cortex cluster and mirror them to a secondary cluster in background. The performances and availability of the secondary cluster have no impact on the requests to the primary one. The response to the client will always be the one from the primary one. In this sense, the requests from Envoy to the secondary cluster are “fire and forget”.\nExample Envoy config The following Envoy configuration shows an example with two Cortex clusters. Envoy will listen on port 9900 and will proxies all requests to cortex-primary:80, mirroring it to cortex-secondary:80 too.\nadmin: # No access logs. access_log_path: /dev/null address: socket_address: { address: 0.0.0.0, port_value: 9901 } static_resources: listeners: - name: cortex_listener address: socket_address: { address: 0.0.0.0, port_value: 9900 } filter_chains: - filters: - name: envoy.http_connection_manager config: stat_prefix: cortex_ingress route_config: name: all_routes virtual_hosts: - name: all_hosts domains: [\"*\"] routes: - match: { prefix: \"/\" } route: cluster: cortex_primary # Specifies the upstream timeout. This spans between the point at which the entire downstream # request has been processed and when the upstream response has been completely processed. timeout: 15s # Specifies the cluster that requests will be mirrored to. The performances and availability of # the secondary cluster have no impact on the requests to the primary one. The response to the # client will always be the one from the primary one. The requests from Envoy to the secondary # cluster are \"fire and forget\". request_mirror_policies: - cluster: cortex_secondary http_filters: - name: envoy.router clusters: - name: cortex_primary type: STRICT_DNS connect_timeout: 1s hosts: [{ socket_address: { address: cortex-primary, port_value: 80 }}] dns_refresh_rate: 5s - name: cortex_secondary type: STRICT_DNS connect_timeout: 1s hosts: [{ socket_address: { address: cortex-secondary, port_value: 80 }}] dns_refresh_rate: 5s ","categories":"","description":"","excerpt":"Requests mirroring (or shadowing) is a technique you can use to mirror …","ref":"/docs/operations/requests-mirroring-to-secondary-cluster/","tags":"","title":"Requests mirroring to secondary cluster"},{"body":"In order to query series inside blocks from object storage, the store-gateway has to know certain initial info from each block index. In order to achieve so, the store-gateway builds an index-header for each block and stores it on local disk; such index-header is built by downloading specific pieces of original block’s index and storing them on local disk. Index header is then used by store-gateway at query time.\nStore-gateways build the index-header with specific sections of the block’s index downloaded using GET byte range requests. Since downloading specific sections of the original block’s index is a computationally easy operation, the index-header is never uploaded back to the object storage and multiple store-gateway instances (or the same instance after a rolling update without a persistent disk) will re-build the index-header from original block’s index each time, if not already existing on local disk.\nFormat (version 1) The index-header is a subset of the block index and contains:\nSymbol Table, used to unintern string values Posting Offset Table, used to lookup postings The following describes the format of the index-header file found in each block store-gateway local directory. It is terminated by a table of contents which serves as an entry point into the index.\n┌─────────────────────────────┬───────────────────────────────┐ │ magic(0xBAAAD792) \u003c4b\u003e │ version(1) \u003c1 byte\u003e │ ├─────────────────────────────┬───────────────────────────────┤ │ index version(2) \u003c1 byte\u003e │ index PostingOffsetTable \u003c8b\u003e │ ├─────────────────────────────┴───────────────────────────────┤ │ ┌─────────────────────────────────────────────────────────┐ │ │ │ Symbol Table (exact copy from original index) │ │ │ ├─────────────────────────────────────────────────────────┤ │ │ │ Posting Offset Table (exact copy from index) │ │ │ ├─────────────────────────────────────────────────────────┤ │ │ │ TOC │ │ │ └─────────────────────────────────────────────────────────┘ │ └─────────────────────────────────────────────────────────────┘ ","categories":"","description":"","excerpt":"In order to query series inside blocks from object storage, the …","ref":"/docs/blocks-storage/binary-index-header/","tags":"","title":"Binary index-header"},{"body":"The bucket index is a per-tenant file containing the list of blocks and block deletion marks in the storage. The bucket index itself is stored in the backend object storage, is periodically updated by the compactor, and used by queriers, store-gateways and rulers to discover blocks in the storage.\nThe bucket index usage is optional and can be enabled via -blocks-storage.bucket-store.bucket-index.enabled=true (or its respective YAML config option).\nBenefits The querier, store-gateway and ruler need to have an almost up-to-date view over the entire storage bucket, in order to find the right blocks to lookup at query time (querier) and load block’s index-header (store-gateway). Because of this, they need to periodically scan the bucket to look for new blocks uploaded by ingester or compactor, and blocks deleted (or marked for deletion) by compactor.\nWhen the bucket index is enabled, the querier, store-gateway and ruler periodically look up the per-tenant bucket index instead of scanning the bucket via “list objects” operations. This brings few benefits:\nReduced number of API calls to the object storage by querier and store-gateway No “list objects” storage API calls done by querier and store-gateway The querier is up and running immediately after the startup (no need to run an initial bucket scan) Structure of the index The bucket-index.json.gz contains:\nblocks\nList of complete blocks of a tenant, including blocks marked for deletion (partial blocks are excluded from the index). block_deletion_marks\nList of block deletion marks. updated_at\nUnix timestamp (seconds precision) of when the index has been updated (written in the storage) the last time. How it gets updated The compactor periodically scans the bucket and uploads an updated bucket index to the storage. The frequency at which the bucket index is updated can be configured via -compactor.cleanup-interval.\nDespite using the bucket index is optional, the index itself is built and updated by the compactor even if -blocks-storage.bucket-store.bucket-index.enabled has not been enabled. This is intentional, so that once a Cortex cluster operator decides to enable the bucket index in a live cluster, the bucket index for any tenant is already existing and query results consistency is guaranteed. The overhead introduced by keeping the bucket index updated is expected to be non significative.\nHow it’s used by the querier At query time the querier and ruler check whether the bucket index for the tenant has already been loaded in memory. If not, the querier and ruler download it from the storage and cache it in memory.\nGiven it’s a small file, lazy downloading it doesn’t significantly impact on first query performances, but allows to get a querier up and running without pre-downloading every tenant’s bucket index. Moreover, if the metadata cache is enabled, the bucket index will be cached for a short time in a shared cache, reducing the actual latency and number of API calls to the object storage in case multiple queriers and rulers will fetch the same tenant’s bucket index in a short time.\nWhile in-memory, a background process will keep it updated at periodic intervals, so that subsequent queries from the same tenant to the same querier instance will use the cached (and periodically updated) bucket index. There are two config options involved:\n-blocks-storage.bucket-store.sync-interval\nThis option configures how frequently a cached bucket index should be refreshed. -blocks-storage.bucket-store.bucket-index.update-on-error-interval\nIf downloading a bucket index fails, the failure is cached for a short time in order to avoid hammering the backend storage. This option configures how frequently a bucket index, which previously failed to load, should be tried to load again. If a bucket index is unused for a long time (configurable via -blocks-storage.bucket-store.bucket-index.idle-timeout), e.g. because that querier instance is not receiving any query from the tenant, the querier will offload it, stopping to keep it updated at regular intervals. This is particularly for tenants which are resharded to different queriers when shuffle sharding is enabled.\nFinally, at query time the querier and ruler check how old a bucket index is (based on its updated_at) and fail a query if its age is older than -blocks-storage.bucket-store.bucket-index.max-stale-period. This circuit breaker is used to ensure queriers and rulers will not return any partial query results due to a stale view over the long-term storage.\nHow it’s used by the store-gateway The store-gateway, at startup and periodically, fetches the bucket index for each tenant belonging to their shard and uses it as the source of truth for the blocks (and deletion marks) in the storage. This removes the need to periodically scan the bucket to discover blocks belonging to their shard.\n","categories":"","description":"","excerpt":"The bucket index is a per-tenant file containing the list of blocks …","ref":"/docs/blocks-storage/bucket-index/","tags":"","title":"Bucket Index"},{"body":"","categories":"","description":"","excerpt":"","ref":"/docs/configuration/","tags":"","title":"Configuration"},{"body":"Cortex integration tests are written in Go and based on a custom framework running Cortex and its dependencies in Docker containers and using the Go testing package for assertions. Integration tests run in CI for every PR, and can be easily executed locally during development (it just requires Docker).\nHow to run integration tests When integration tests run in CI, we build the Cortex docker image based on the PR code and then run the integration tests against it. When running tests locally you should build the Cortex Docker image first:\nmake ./cmd/cortex/.uptodate This will locally build the quay.io/cortexproject/cortex:latest image used by integration tests. Whenever the Cortex code changes (cmd/, pkg/ or vendors) you should rebuild the Cortex image, while it’s not necessary to rebuild it while developing integration tests.\nOnce the Docker image is built, you can run integration tests:\ngo test -v -tags=integration,requires_docker,integration_alertmanager,integration_memberlist,integration_querier,integration_ruler,integration_query_fuzz ./integration/... If you want to run a single test you can use a filter. For example, to only run TestRulerAPISharding:\ngo test -v -tags=integration,integration_ruler -timeout 2400s -v -count=1 ./integration/... -run \"^TestRulerAPISharding$\" Supported environment variables CORTEX_IMAGE\nDocker image used to run Cortex in integration tests (defaults to quay.io/cortexproject/cortex:latest) CORTEX_CHECKOUT_DIR\nThe absolute path of the Cortex repository local checkout (defaults to $GOPATH/src/github.com/cortexproject/cortex) E2E_TEMP_DIR\nThe absolute path to a directory where the integration test will create an additional temporary directory to store files generated during the test. E2E_NETWORK_NAME\nName of the docker network to create and use for integration tests. If no variable is set, defaults to e2e-cortex-test. The requires_docker tag Integration tests have requires_docker tag (// +build requires_docker line followed by empty line on top of Go files), to avoid running them unintentionally as they require Docker, e.g. by running go test ./... in main Cortex package.\nIsolation Each integration test runs in isolation. For each integration test, we do create a Docker network, start Cortex and its dependencies containers, push/query series to/from Cortex and run assertions on it. Once the test has done, both the Docker network and containers are terminated and deleted.\n","categories":"","description":"","excerpt":"Cortex integration tests are written in Go and based on a custom …","ref":"/docs/contributing/how-integration-tests-work/","tags":"","title":"How integration tests work"},{"body":"The procedure is:\nUpdate build-image/Dockerfile Create a PR to master with that changed, after the PR is merged to master, the new build image is available in the quay.io repository. Check github action logs here for to find the image tag. Create another PR to replace the image tag in .github/workflows/* (there may be multiple references) and Makefile (variable LATEST_BUILD_IMAGE_TAG). If you are updating Go’s runtime version be sure to change actions/setup-go’s go-version in ``.github/workflows/*`. Open a PR and make sure the CI with new build-image passes ","categories":"","description":"","excerpt":"The procedure is:\nUpdate build-image/Dockerfile Create a PR to master …","ref":"/docs/contributing/how-to-update-the-build-image/","tags":"","title":"How to update the build image"},{"body":"This article describes how to migrate existing Cortex cluster from chunks storage to blocks storage, and highlight possible issues you may encounter in the process.\nThis document replaces the Cortex proposal, which was written before support for migration was in place.\nIntroduction This article assumes that:\nCortex cluster is managed by Kubernetes Cortex is using chunks storage Ingesters are using WAL Cortex version 1.4.0 or later. If your ingesters are not using WAL, the documented procedure will still apply, but the presented migration script will not work properly without changes, as it assumes that ingesters are managed via StatefulSet.\nThe migration procedure is composed by 3 steps:\nPreparation Ingesters migration Cleanup In case of any issue during or after the migration, this document also outlines a Rollback strategy.\nStep 1: Preparation Before starting the migration of ingesters, we need to prepare other services.\nQuerier and Ruler Everything discussed for querier applies to ruler as well, since it shares querier configuration – CLI flags prefix is -querier even when used by ruler.\nQuerier and ruler need to be reconfigured as follow:\n-querier.second-store-engine=blocks -querier.query-store-after=0 -querier.second-store-engine=blocks Querier (and ruler) needs to be reconfigured to query both chunks storage and blocks storage at the same time. This is achieved by using -querier.second-store-engine=blocks option, and providing querier with full blocks configuration, but keeping “primary” store set to -store.engine=chunks.\n-querier.query-store-after=0 Querier (and ruler) has an option -querier.query-store-after to query store only if query hits data older than some period of time. For example, if ingesters keep 12h of data in memory, there is no need to hit the store for queries that only need last 1h of data. During the migration, this flag needs to be set to 0, to make queriers always consult the store when handling queries. As chunks ingesters shut down, they flush chunks to the storage. They are then replaced with new ingesters configured to use blocks. Queriers cannot fetch recent chunks from ingesters directly (as blocks ingester don’t reload chunks), and need to use storage instead.\nCompactor and Store-gateway Compactor and store-gateway services should be deployed and successfully up and running before migrating ingesters.\nIngester – blocks Migration script presented in Step 2 assumes that there are two StatefulSets of ingesters: existing one configured with chunks, and the new one with blocks. New StatefulSet with blocks ingesters should have 0 replicas at the beginning of migration.\nTable-Manager - chunks If you use a store with provisioned IO, e.g. DynamoDB, scale up the provision before starting the migration. Each ingester will need to flush all chunks before exiting, so will write to the store at many times the normal rate.\nStop or reconfigure the table-manager to stop it adjusting the provision back to normal. (Don’t do the migration on Wednesday night when a new weekly table might be required.)\nStep 2: Ingesters migration We have developed a script available in Cortex tools/migrate-ingester-statefulsets.sh to migrate ingesters between two StatefulSets, shutting down ingesters one by one.\nIt can be used like this:\n$ tools/migrate-ingester-statefulsets.sh \u003cnamespace\u003e \u003cingester-old\u003e \u003cingester-new\u003e \u003cnum-instances\u003e Where parameters are:\n\u003cnamespace\u003e: Kubernetes namespace where the Cortex cluster is running \u003cingester-old\u003e: name of the ingesters StatefulSet to scale down (running chunks storage) \u003cingester-new\u003e: name of the ingesters StatefulSet to scale up (running blocks storage) \u003cnum-instances\u003e: number of instances to scale down (in ingester-old statefulset) and scale up (in ingester-new), or “all” – which will scale down all remaining instances in ingester-old statefulset After starting new pod in ingester-new statefulset, script then triggers /shutdown endpoint on the old ingester. When the flushing on the old ingester is complete, scale down of statefulset continues, and process repeats.\nThe script supports both migration from chunks to blocks, and vice versa (eg. rollback).\nKnown issues There are few known issues with the script:\nIf expected messages don’t appear in the log, but pod keeps on running, the script will never finish. Script doesn’t verify that flush finished without any error. Step 3: Cleanup When the ingesters migration finishes, there are still two StatefulSets, with original StatefulSet (running the chunks storage) having 0 instances now.\nAt this point, we can delete the old StatefulSet and its persistent volumes and recreate it with final blocks storage configuration (eg. changing PVs), and use the script again to move pods from ingester-blocks to ingester.\nQuerier (and ruler) can be reconfigured to use blocks as “primary” store to search, and chunks as secondary:\n-store.engine=blocks -querier.second-store-engine=chunks -querier.use-second-store-before-time=\u003ctimestamp after ingesters migration has completed\u003e -querier.ingester-streaming=true -querier.use-second-store-before-time The CLI flag -querier.use-second-store-before-time (or its respective YAML config option) is only available for secondary store. This flag can be set to a timestamp when migration has finished, and it avoids querying secondary store (chunks) for data when running queries that don’t need data before given time.\nBoth primary and secondary stores are queried before this time, so the overlap where some data is in chunks and some in blocks is covered.\nRollback If rollback to chunks is needed for any reason, it is possible to use the same migration script with reversed arguments:\nScale down ingesters StatefulSet running blocks storage Scale up ingesters StatefulSet running chunks storage Blocks ingesters support the same /shutdown endpoint for flushing data.\nDuring the rollback, queriers and rulers need to use the same configuration changes as during migration. You should also make sure the following settings are applied:\n-store.engine=chunks -querier.second-store-engine=blocks -querier.use-second-store-before-time should not be set -querier.ingester-streaming=false Once the rollback is complete, some configuration changes need to stay in place, because some data has already been stored to blocks:\nThe query sharding in the query-frontend must be kept disabled, otherwise querying blocks will not work correctly store-gateway needs to keep running, otherwise querying blocks will fail compactor may be shutdown, after it has no more compaction work to do Kubernetes resources related to the ingesters running the blocks storage may be deleted.\nKnown issues After rollback, chunks ingesters will replay their old Write-Ahead-Log, thus loading old chunks into memory. WAL doesn’t remember whether these old chunks were already flushed or not, so they will be flushed again to the storage. Until that flush happens, Cortex reports those chunks as unflushed, which may trigger some alerts based on cortex_oldest_unflushed_chunk_timestamp_seconds metric.\nAppendix Jsonnet config This section shows how to use cortex-jsonnet to configure additional services.\nWe will assume that main.jsonnet is main configuration for the cluster, that also imports temp.jsonnet – with our temporary configuration for migration.\nIn main.jsonnet we have something like this:\nlocal cortex = import 'cortex/cortex.libsonnet'; local wal = import 'cortex/wal.libsonnet'; local temp = import 'temp.jsonnet'; // Note that 'tsdb' is not imported here. cortex + wal + temp { _images+:: (import 'images.libsonnet'), _config+:: { cluster: 'k8s-cluster', namespace: 'k8s-namespace', ... To configure querier to use secondary store for querying, we need to add:\nquerier_second_storage_engine: 'blocks', blocks_storage_bucket_name: 'bucket-for-storing-blocks', to the _config object in main.jsonnet.\nLet’s generate blocks configuration now in temp.jsonnet. There are comments inside that should give you an idea about what’s happening. Most important thing is generating resources with blocks configuration, and exposing some of them.\n{ local cortex = import 'cortex/cortex.libsonnet', local tsdb = import 'cortex/tsdb.libsonnet', local rootConfig = self._config, local statefulSet = $.apps.v1beta1.statefulSet, // Prepare TSDB resources, but hide them. Cherry-picked resources will be exposed later. tsdb_config:: cortex + tsdb + { _config+:: { cluster: rootConfig.cluster, namespace: rootConfig.namespace, external_url: rootConfig.external_url, // This Cortex cluster is using the blocks storage. storage_tsdb_bucket_name: rootConfig.storage_tsdb_bucket_name, cortex_store_gateway_data_disk_size: '100Gi', cortex_compactor_data_disk_class: 'fast', }, // We create another statefulset for ingesters here, with different name. ingester_blocks_statefulset: self.newIngesterStatefulSet('ingester-blocks', self.ingester_container) + statefulSet.mixin.spec.withReplicas(0), ingester_blocks_pdb: self.newIngesterPdb('ingester-blocks-pdb', 'ingester-blocks'), ingester_blocks_service: $.util.serviceFor(self.ingester_blocks_statefulset, self.ingester_service_ignored_labels), }, _config+: { queryFrontend+: { // Disabled because querying blocks-data breaks if query is rewritten for sharding. sharded_queries_enabled: false, }, }, // Expose some services from TSDB configuration, needed for running Querier with Chunks as primary and TSDB as secondary store. tsdb_store_gateway_pdb: self.tsdb_config.store_gateway_pdb, tsdb_store_gateway_service: self.tsdb_config.store_gateway_service, tsdb_store_gateway_statefulset: self.tsdb_config.store_gateway_statefulset, tsdb_memcached_metadata: self.tsdb_config.memcached_metadata, tsdb_ingester_statefulset: self.tsdb_config.ingester_blocks_statefulset, tsdb_ingester_pdb: self.tsdb_config.ingester_blocks_pdb, tsdb_ingester_service: self.tsdb_config.ingester_blocks_service, tsdb_compactor_statefulset: self.tsdb_config.compactor_statefulset, // Querier and ruler configuration used during migration, and after. query_config_during_migration:: { // Disable streaming, as it is broken when querying both chunks and blocks ingesters at the same time. 'querier.ingester-streaming': 'false', // query-store-after is required during migration, since new ingesters running on blocks will not load any chunks from chunks-WAL. // All such chunks are however flushed to the store. 'querier.query-store-after': '0', }, query_config_after_migration:: { 'querier.ingester-streaming': 'true', 'querier.query-ingesters-within': '13h', // TSDB ingesters have data for up to 4d. 'querier.query-store-after': '12h', // Can be enabled once blocks ingesters are running for 12h. // Switch TSDB and chunks. TSDB is \"primary\" now so that we can skip querying chunks for old queries. // We can do this, because querier/ruler have both configurations. 'store.engine': 'blocks', 'querier.second-store-engine': 'chunks', 'querier.use-second-store-before-time': '2020-07-28T17:00:00Z', // If migration from chunks finished around 18:10 CEST, no need to query chunk store for queries before this time. }, querier_args+:: self.tsdb_config.blocks_metadata_caching_config + self.query_config_during_migration, // + self.query_config_after_migration, ruler_args+:: self.tsdb_config.blocks_metadata_caching_config + self.query_config_during_migration, // + self.query_config_after_migration, } ","categories":"","description":"","excerpt":"This article describes how to migrate existing Cortex cluster from …","ref":"/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/","tags":"","title":"Migrate Cortex cluster from chunks to blocks"},{"body":"Historically scaling the Cortex query frontend has posed some challenges. This document aims to detail how to use some added configuration parameters to correctly scale the frontend. Note that these instructions apply in both the HA single binary scenario or microservices mode.\nScaling the Query Frontend For every query frontend the querier adds a configurable number of concurrent workers which are each capable of executing a query. Each worker is connected to a single query frontend instance, therefore scaling up the query frontend impacts the amount of work each individual querier is attempting to do at any given time.\nScaling up may cause a querier to attempt more work than they are configured for due to restrictions such as memory and cpu limits. Additionally, the PromQL engine itself is limited in the number of queries it can do as configured by the -querier.max-concurrent parameter. Attempting more queries concurrently than this value causes the queries to queue up in the querier itself.\nFor similar reasons scaling down the query frontend may cause a querier to not use its allocated memory and cpu effectively. This will lower effective resource utilization. Also, because individual queriers will be doing less work, this may cause increased queueing in the query frontends.\nQuerier Max Concurrency To guarantee that querier doesn’t receive more queries that it can handle at the same time, make sure to configure the querier to match its PromQL concurrency with number of connections. This can be done by using -querier.worker-match-max-concurrent=true option, or match_max_concurrent: true field in frontend_worker section of YAML config file. This allows the operator to freely scale the frontend or scheduler up and down without impacting the amount of work an individual querier is attempting to perform.\nQuery Scheduler Query scheduler is a service that moves the in-memory queue from query frontend to a separate component. This makes scaling query frontend easier, as it allows running multiple query frontends without increasing the number of queues.\nIn order to use query scheduler, both query frontend and queriers must be configured with query scheduler address (using -frontend.scheduler-address and -querier.scheduler-address options respectively).\nNote that querier will only fetch queries from query frontend or query scheduler, but not both. -querier.frontend-address and -querier.scheduler-address options are mutually exclusive, and at most one can be set.\nWhen using query scheduler, it is recommended to run two query scheduler instances. Running only one query scheduler poses a risk of increased query latency when single scheduler crashes or restarts. Running two query-schedulers should be enough even for large Cortex clusters with an high QPS.\nWhen using single-binary mode, Cortex defaults to run without query scheduler.\nDNS Configuration / Readiness When a new frontend is first created on scale up it will not immediately have queriers attached to it. The existing endpoint /ready returns HTTP 200 status code only when the query frontend is ready to serve queries. Make sure to configure this endpoint as a healthcheck in your load balancer, otherwise a query frontend scale up event might result in failed queries or high latency for a bit while queriers attach.\nWhen using query frontend with query scheduler, /ready will report 200 status code only after frontend discovers some schedulers via DNS resolution.\n","categories":"","description":"","excerpt":"Historically scaling the Cortex query frontend has posed some …","ref":"/docs/operations/scaling-query-frontend/","tags":"","title":"Scaling the Query Frontend"},{"body":"If you have configured your cluster to write new data to blocks, there is still a question about old data. Cortex can query both chunks and the blocks at the same time, but converting old chunks to blocks still has some benefits, like being able to decommission the chunks storage backend and save costs. This document presents set of tools for doing the conversion.\nOriginal design document for blocksconvert is also available.\nTools Cortex provides a tool called blocksconvert, which is actually collection of three tools for converting chunks to blocks.\nTools are:\nScanner\nScans the chunks index database and produces so-called “plan files”, each file being a set of series and chunks for each series. Plan files are uploaded to the same object store bucket where blocks live. Scheduler\nLooks for plan files, and distributes them to builders. Scheduler has global view of overall conversion progress. Builder\nAsks scheduler for next plan file to work on, fetches chunks, puts them into TSDB block, and uploads the block to the object store. It repeats this process until there are no more plans. Cleaner\nCleaner asks scheduler for next plan file to work on, but instead of building the block, it actually REMOVES CHUNKS and INDEX ENTRIES from the Index database. All tools start HTTP server (see -server.http* options) exposing the /metrics endpoint. All tools also start gRPC server (-server.grpc* options), but only Scheduler exposes services on it.\nScanner Scanner is started by running blocksconvert -target=scanner. Scanner requires configuration for accessing Cortex Index:\n-schema-config-file – this is standard Cortex schema file. -bigtable.instance, -bigtable.project – options for BigTable access. -dynamodb.url - for DynamoDB access. Example dynamodb://us-east-1/ -blocks-storage.backend and corresponding -blocks-storage.* options for storing plan files. -scanner.output-dir – specifies local directory for writing plan files to. Finished plan files are deleted after upload to the bucket. List of scanned tables is also kept in this directory, to avoid scanning the same tables multiple times when Scanner is restarted. -scanner.allowed-users – comma-separated list of Cortex tenants that should have plans generated. If empty, plans for all found users are generated. -scanner.ignore-users-regex - If plans for all users are generated (-scanner.allowed-users is not set), then users matching this non-empty regular expression will be skipped. -scanner.tables-limit – How many tables should be scanned? By default all tables are scanned, but when testing scanner it may be useful to start with small number of tables first. -scanner.tables – Comma-separated list of tables to be scanned. Can be used to scan specific tables only. Note that schema is still used to find all tables first, and then this list is consulted to select only specified tables. -scanner.scan-period-start \u0026 -scanner.scan-period-end - limit the scan to a particular date range (format like 2020-12-31) Scanner will read the Cortex schema file to discover Index tables, and then it will start scanning them from most-recent table first, going back. For each table, it will fully read the table and generate a plan for each user and day stored in the table. Plan files are then uploaded to the configured blocks-storage bucket (at the -blocksconvert.bucket-prefix location prefix), and local copies are deleted. After that, scanner continues with the next table until it scans them all or -scanner.tables-limit is reached.\nNote that even though blocksconvert has options for configuring different Index store backends, it only supports BigTable and DynamoDB at the moment.\nIt is expected that only single Scanner process is running. Scanner does the scanning of multiple table subranges concurrently.\nScanner exposes metrics with cortex_blocksconvert_scanner_ prefix, eg. total number of scanned index entries of different type, number of open files (scanner doesn’t close currently plan files until entire table has been scanned), scanned rows and parsed index entries.\nScanner only supports schema version v9 on DynamoDB; v9, v10 and v11 on BigTable. Earlier schema versions are currently not supported.\nScheduler Scheduler is started by running blocksconvert -target=scheduler. It only needs to be configured with options to access the object store with blocks:\n-blocks-storage.* - Blocks storage object store configuration. -scheduler.scan-interval – How often to scan for plan files and their status. -scheduler.allowed-users – Comma-separated list of Cortex tenants. If set, only plans for these tenants will be offered to Builders. It is expected that only single Scheduler process is running. Schedulers consume very little resources.\nScheduler’s metrics have cortex_blocksconvert_scheduler prefix (number of plans in different states, oldest/newest plan). Scheduler HTTP server also exposes /plans page that shows currently queued plans, and all plans and their status for all users.\nBuilder Builder asks scheduler for next plan to work on, downloads the plan, builds the block and uploads the block to the blocks storage. It then repeats the process while there are still plans.\nBuilder is started by blocksconvert -target=builder. It needs to be configured with Scheduler endpoint, Cortex schema file, chunk-store specific options and blocks storage to upload blocks to.\n-builder.scheduler-endpoint - where to find scheduler, eg. “scheduler:9095” -schema-config-file - Cortex schema file, used to find out which chunks store to use for given plan -gcs.bucketname – when using GCS as chunks store (other chunks backend storages, like S3, are supported as well) -blocks-storage.* - blocks storage configuration -builder.output-dir - Local directory where Builder keeps the block while it is being built. Once block is uploaded to blocks storage, it is deleted from local directory. Multiple builders may run at the same time, each builder will receive different plan to work on from scheduler. Builders are CPU intensive (decoding and merging chunks), and require fast disk IO for writing blocks.\nBuilders’s metrics have cortex_blocksconvert_builder prefix, and include total number of fetched chunks and their size, read position of the current plan and plan size, total number of written series and samples, number of chunks that couldn’t be downloaded.\nCleaner Cleaner is similar to builder in that it asks scheduler for next plan to work on, but instead of building blocks, it actually REMOVES CHUNKS and INDEX ENTRIES. Use with caution.\nCleaner is started by using blocksconvert -target=cleaner. Like Builder, it needs Scheduler endpoint, Cortex schema file, index and chunk-store specific options. Note that Cleaner works with any index store supported by Cortex, not just BigTable.\n-cleaner.scheduler-endpoint – where to find scheduler -blocks-storage.* – blocks storage configuration, used for downloading plan files -cleaner.plans-dir – local directory to store plan file while it is being processed by Cleaner. -schema-config-file – Cortex schema file. Cleaner doesn’t scan for index entries, but uses existing plan files to find chunks and index entries. For each series, Cleaner needs to download at least one chunk. This is because plan file doesn’t contain label names and values, but chunks do. Cleaner will then delete all index entries associated with the series, and also all chunks.\nWARNING: If both Builder and Cleaner run at the same time and use use the same Scheduler, some plans will be handled by builder, and some by cleaner! This will result in a loss of data!\nCleaner should only be deployed if no other Builder is running. Running multiple Cleaners at once is not supported, and will result in leftover chunks and index entries. Reason for this is that chunks can span multiple days, and chunk is fully deleted only when processing plan (day) when chunk started. Since cleaner also needs to download some chunks to be able to clean up all index entries, when using multiple cleaners, it can happen that cleaner processing older plans will delete chunks required to properly clean up data in newer plans. When using single cleaner only, this is not a problem, since scheduler sends plans to cleaner in time-reversed order.\nNote: Cleaner is designed for use in very special cases, eg. when deleting chunks and index entries for a specific customer. If blocksconvert was used to convert ALL chunks to blocks, it is simpler to just drop the index and chunks database afterwards. In such case, Cleaner is not needed.\nLimitations The blocksconvert toolset currently has the following limitations:\nScanner supports only BigTable and DynamoDB for chunks index backend, and cannot currently scan other databases. Supports only chunks schema versions v9 for DynamoDB; v9, v10 and v11 for Bigtable. ","categories":"","description":"","excerpt":"If you have configured your cluster to write new data to blocks, there …","ref":"/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/","tags":"","title":"Convert long-term storage from chunks to blocks"},{"body":"","categories":"","description":"","excerpt":"","ref":"/docs/guides/","tags":"","title":"Guides"},{"body":"For the v1.0 release, we want to provide the following guarantees:\nFlags, Config and minor version upgrades Upgrading cortex from one minor version to the next should “just work”; that being said, we don’t want to bump the major version every time we remove a flag, so we will will keep deprecated flags around for 2 minor release. There is a metric (deprecated_flags_inuse_total) you can alert on to find out if you’re using a deprecated flag.\nSimilarly to flags, minor version upgrades using config files should “just work”. If we do need to change config, we will keep the old way working for two minor version. There will be a metric you can alert on for this too.\nThese guarantees don’t apply for experimental features.\nReading old data The Cortex maintainers commit to ensuring future version of Cortex can read data written by versions up to two years old. In practice we expect to be able to read more, but this is our guarantee.\nAPI Compatibility Cortex strives to be 100% API compatible with Prometheus (under /prometheus/* and /api/prom/*); any deviation from this is considered a bug, except:\nAdditional API endpoints for creating, removing and modifying alerts and recording rules. Additional API around pushing metrics (under /api/push). Additional API endpoints for management of Cortex itself, such as the ring. These APIs are not part of the any compatibility guarantees. For more information, please refer to the limitations doc.\nExperimental features Cortex is an actively developed project and we want to encourage the introduction of new features and capability. As such, not everything in each release of Cortex is considered “production-ready”. We don’t provide any backwards compatibility guarantees on these and the config and flags might break.\nCurrently experimental features are:\nRuler: Evaluate rules to query frontend instead of ingesters (enabled via -ruler.frontend-address ) S3 Server Side Encryption (SSE) using KMS (including per-tenant KMS config overrides). Azure blob storage. Zone awareness based replication. Ruler API (to PUT rules). Alertmanager: API (enabled via -experimental.alertmanager.enable-api) Sharding of tenants across multiple instances (enabled via -alertmanager.sharding-enabled) Receiver integrations firewall (configured via -alertmanager.receivers-firewall.*) Memcached client DNS-based service discovery. In-memory (FIFO) and Redis cache. gRPC Store. TLS configuration in gRPC and HTTP clients. TLS configuration in Etcd client. Blocksconvert tools OpenStack Swift storage support. Metric relabeling in the distributor. Scalable query-frontend (when using query-scheduler) Ingester: do not unregister from ring on shutdown (-ingester.unregister-on-shutdown=false) Distributor: Do not extend writes on unhealthy ingesters (-distributor.extend-writes=false) Accept multiple HA pairs in the same request (enabled via -experimental.distributor.ha-tracker.mixed-ha-samples=true) Tenant Deletion in Purger, for blocks storage. Query-frontend: query stats tracking (-frontend.query-stats-enabled) Blocks storage bucket index The bucket index support in the querier and store-gateway (enabled via -blocks-storage.bucket-store.bucket-index.enabled=true) is experimental The block deletion marks migration support in the compactor (-compactor.block-deletion-marks-migration-enabled) is temporarily and will be removed in future versions Querier: tenant federation The thanosconvert tool for converting Thanos block metadata to Cortex HA Tracker: cleanup of old replicas from KV Store. Instance limits in ingester and distributor Exemplar storage, currently in-memory only within the Ingester based on Prometheus exemplar storage (-blocks-storage.tsdb.max-exemplars) Querier limits: -querier.max-fetched-chunks-per-query -querier.max-fetched-chunk-bytes-per-query -querier.max-fetched-series-per-query Alertmanager limits notification rate (-alertmanager.notification-rate-limit and -alertmanager.notification-rate-limit-per-integration) dispatcher groups (-alertmanager.max-dispatcher-aggregation-groups) user config size (-alertmanager.max-config-size-bytes) templates count in user config (-alertmanager.max-templates-count) max template size (-alertmanager.max-template-size-bytes) Disabling ring heartbeat timeouts -distributor.ring.heartbeat-timeout=0 -ring.heartbeat-timeout=0 -ruler.ring.heartbeat-timeout=0 -alertmanager.sharding-ring.heartbeat-timeout=0 -compactor.ring.heartbeat-timeout=0 -store-gateway.sharding-ring.heartbeat-timeout=0 Disabling ring heartbeats -distributor.ring.heartbeat-period=0 -ingester.heartbeat-period=0 -ruler.ring.heartbeat-period=0 -alertmanager.sharding-ring.heartbeat-period=0 -compactor.ring.heartbeat-period=0 -store-gateway.sharding-ring.heartbeat-period=0 Compactor shuffle sharding Enabled via -compactor.sharding-enabled=true, -compactor.sharding-strategy=shuffle-sharding, and -compactor.tenant-shard-size set to a value larger than 0. Vertical sharding at query frontend for range/instant queries -frontend.query-vertical-shard-size (int) CLI flag query_vertical_shard_size (int) field in runtime config file Snapshotting of in-memory TSDB on disk during shutdown -blocks-storage.tsdb.memory-snapshot-on-shutdown (boolean) CLI flag Out of order samples support -blocks-storage.tsdb.out-of-order-cap-max (int) CLI flag -ingester.out-of-order-time-window (duration) CLI flag out_of_order_time_window (duration) field in runtime config file Store Gateway Zone Stable Shuffle Sharding -store-gateway.sharding-ring.zone-stable-shuffle-sharding CLI flag zone_stable_shuffle_sharding (boolean) field in config file Basic Lifecycler (Storegateway, Alertmanager, Ruler) Final Sleep on shutdown, which tells the pod wait before shutdown, allowing a delay to propagate ring changes. -ruler.ring.final-sleep (duration) CLI flag store-gateway.sharding-ring.final-sleep (duration) CLI flag alertmanager-sharding-ring.final-sleep (duration) CLI flag OTLP Receiver Persistent tokens in the Ruler Ring: -ruler.ring.tokens-file-path (path) CLI flag Native Histograms Ingestion can be enabled by setting -blocks-storage.tsdb.enable-native-histograms=true on Ingester. String interning for metrics labels Enable string interning for metrics labels by setting -ingester.labels-string-interning-enabled on Ingester. Query-frontend: query rejection (-frontend.query-rejection.enabled) Querier: protobuf codec (-api.querier-default-codec) ","categories":"","description":"","excerpt":"For the v1.0 release, we want to provide the following guarantees: …","ref":"/docs/configuration/v1guarantees/","tags":"","title":"v1.x Guarantees"},{"body":"Cortex exposes an HTTP API for pushing and querying time series data, and operating the cluster itself.\nFor the sake of clarity, in this document we have grouped API endpoints by service, but keep in mind that they’re exposed both when running Cortex in microservices and singly-binary mode:\nMicroservices: each service exposes its own endpoints Single-binary: the Cortex process exposes all API endpoints for the services running internally Endpoints API Service Available Since Endpoint Index page All services GET / Configuration All services GET /config Runtime Configuration All services GET /runtime_config Services status All services GET /services Readiness probe All services GET /ready Metrics All services GET /metrics Pprof All services GET /debug/pprof Fgprof All services GET /debug/fgprof Remote write Distributor POST /api/v1/push OTLP receiver Distributor POST /api/v1/otlp/v1/metrics Tenants stats Distributor GET /distributor/all_user_stats HA tracker status Distributor GET /distributor/ha_tracker Flush blocks Ingester GET,POST /ingester/flush Shutdown Ingester GET,POST /ingester/shutdown Ingesters ring status Ingester GET /ingester/ring Ingester tenants stats Ingester GET /ingester/all_user_stats Ingester mode Ingester GET,POST /ingester/mode Instant query Querier, Query-frontend GET,POST \u003cprometheus-http-prefix\u003e/api/v1/query Range query Querier, Query-frontend GET,POST \u003cprometheus-http-prefix\u003e/api/v1/query_range Exemplar query Querier, Query-frontend GET,POST \u003cprometheus-http-prefix\u003e/api/v1/query_exemplars Get series by label matchers Querier, Query-frontend GET,POST \u003cprometheus-http-prefix\u003e/api/v1/series Get label names Querier, Query-frontend GET,POST \u003cprometheus-http-prefix\u003e/api/v1/labels Get label values Querier, Query-frontend GET \u003cprometheus-http-prefix\u003e/api/v1/label/{name}/values Get metric metadata Querier, Query-frontend GET \u003cprometheus-http-prefix\u003e/api/v1/metadata Remote read Querier, Query-frontend POST \u003cprometheus-http-prefix\u003e/api/v1/read Build information Querier, Query-frontend v1.15.0 GET \u003cprometheus-http-prefix\u003e/api/v1/status/buildinfo Get tenant ingestion stats Querier GET /api/v1/user_stats Ruler ring status Ruler GET /ruler/ring Ruler rules Ruler GET /ruler/rule_groups List rules Ruler GET \u003cprometheus-http-prefix\u003e/api/v1/rules List alerts Ruler GET \u003cprometheus-http-prefix\u003e/api/v1/alerts List rule groups Ruler GET /api/v1/rules Get rule groups by namespace Ruler GET /api/v1/rules/{namespace} Get rule group Ruler GET /api/v1/rules/{namespace}/{groupName} Set rule group Ruler POST /api/v1/rules/{namespace} Delete rule group Ruler DELETE /api/v1/rules/{namespace}/{groupName} Delete namespace Ruler DELETE /api/v1/rules/{namespace} Delete tenant configuration Ruler POST /ruler/delete_tenant_config Alertmanager status Alertmanager GET /multitenant_alertmanager/status Alertmanager configs Alertmanager GET /multitenant_alertmanager/configs Alertmanager ring status Alertmanager GET /multitenant_alertmanager/ring Alertmanager UI Alertmanager GET /\u003calertmanager-http-prefix\u003e Alertmanager Delete Tenant Configuration Alertmanager POST /multitenant_alertmanager/delete_tenant_config Get Alertmanager configuration Alertmanager GET /api/v1/alerts Set Alertmanager configuration Alertmanager POST /api/v1/alerts Delete Alertmanager configuration Alertmanager DELETE /api/v1/alerts Tenant delete request Purger POST /purger/delete_tenant Tenant delete status Purger GET /purger/delete_tenant_status Store-gateway ring status Store-gateway GET /store-gateway/ring Compactor ring status Compactor GET /compactor/ring Get rule files Configs API (deprecated) GET /api/prom/configs/rules Set rule files Configs API (deprecated) POST /api/prom/configs/rules Get template files Configs API (deprecated) GET /api/prom/configs/templates Set template files Configs API (deprecated) POST /api/prom/configs/templates Get Alertmanager config file Configs API (deprecated) GET /api/prom/configs/alertmanager Set Alertmanager config file Configs API (deprecated) POST /api/prom/configs/alertmanager Validate Alertmanager config Configs API (deprecated) POST /api/prom/configs/alertmanager/validate Deactivate configs Configs API (deprecated) DELETE /api/prom/configs/deactivate Restore configs Configs API (deprecated) POST /api/prom/configs/restore Path prefixes In this documentation you will find the usage of some placeholders for the path prefixes, whenever the prefix is configurable. The following table shows the supported prefixes.\nPrefix Default CLI Flag YAML Config \u003clegacy-http-prefix\u003e /api/prom -http.prefix http_prefix \u003cprometheus-http-prefix\u003e /prometheus -http.prometheus-http-prefix api \u003e prometheus_http_prefix \u003calertmanager-http-prefix\u003e /alertmanager -http.alertmanager-http-prefix api \u003e alertmanager_http_prefix Authentication When multi-tenancy is enabled, endpoints requiring authentication are expected to be called with the X-Scope-OrgID HTTP request header set to the tenant ID. Otherwise, when multi-tenancy is disabled, Cortex doesn’t require any request to have the X-Scope-OrgID header.\nMulti-tenancy can be enabled/disabled via the CLI flag -auth.enabled or its respective YAML config option.\nFor more information, please refer to the dedicated Authentication and Authorisation guide.\nAll services The following API endpoints are exposed by all services.\nIndex page GET / Displays an index page with links to other web pages exposed by Cortex.\nConfiguration GET /config Displays the configuration currently applied to Cortex (in YAML format), including default values and settings via CLI flags. Sensitive data is masked. Please be aware that the exported configuration doesn’t include the per-tenant overrides.\nDifferent modes GET /config?mode=diff Displays the configuration currently applied to Cortex (in YAML format) as before, but containing only the values that differ from the default values.\nGET /config?mode=defaults Displays the configuration using only the default values.\nRuntime Configuration GET /runtime_config Displays the runtime configuration currently applied to Cortex (in YAML format), including default values. Please be aware that the endpoint will be only available if Cortex is configured with the -runtime-config.file option.\nDifferent modes GET /runtime_config?mode=diff Displays the runtime configuration currently applied to Cortex (in YAML format) as before, but containing only the values that differ from the default values.\nServices status GET /services Displays a web page with the status of internal Cortex services.\nReadiness probe GET /ready Returns 200 when Cortex is ready to serve traffic.\nMetrics GET /metrics Returns the metrics for the running Cortex service in the Prometheus exposition format.\nPprof GET /debug/pprof/heap GET /debug/pprof/block GET /debug/pprof/profile GET /debug/pprof/trace GET /debug/pprof/goroutine GET /debug/pprof/mutex Returns the runtime profiling data in the format expected by the pprof visualization tool. There are many things which can be profiled using this including heap, trace, goroutine, etc.\nFor more information, please check out the official documentation of pprof.\nFgprof GET /debug/fgprof Returns the sampling Go profiling data which allows you to analyze On-CPU as well as Off-CPU (e.g. I/O) time together.\nFor more information, please check out the official documentation of fgprof.\nDistributor Remote write POST /api/v1/push # Legacy POST \u003clegacy-http-prefix\u003e/push Entrypoint for the Prometheus remote write.\nThis API endpoint accepts an HTTP POST request with a body containing a request encoded with Protocol Buffers and compressed with Snappy. The definition of the protobuf message can be found in cortex.proto. The HTTP request should contain the header X-Prometheus-Remote-Write-Version set to 0.1.0.\nFor more information, please check out Prometheus Remote storage integrations.\nRequires authentication.\nOTLP Receiver POST /api/v1/otlp/v1/metrics Entrypoint for the OTLP Receiver\nThis API endpoint accepts a HTTP POST request using OTLP format\nRequires authentication.\nDistributor ring status GET /distributor/ring Displays a web page with the distributor hash ring status, including the state, healthy and last heartbeat time of each distributor.\nTenants stats GET /distributor/all_user_stats # Legacy GET /all_user_stats Displays a web page with per-tenant statistics updated in realtime, including the total number of loaded blocks and active series across all ingesters as well as the current ingestion rate (samples / sec).\nHA tracker status GET /distributor/ha_tracker # Legacy GET /ha-tracker Displays a web page with the current status of the HA tracker, including the elected replica for each Prometheus HA cluster.\nIngester Flush blocks GET,POST /ingester/flush # Legacy GET,POST /flush Triggers a flush of the in-memory time series data to the long-term storage. This endpoint triggers the flush also when -ingester.flush-on-shutdown-with-wal-enabled or -blocks-storage.tsdb.flush-blocks-on-shutdown are disabled.\nThis endpoint accepts tenant parameter to specify tenant whose blocks are compacted and shipped. This parameter may be specified multiple times to select more tenants. If no tenant is specified, all tenants are flushed.\nFlush endpoint now also accepts wait=true parameter, which makes the call synchronous – it will only return after flushing has finished. Note that returned status code does not reflect the result of flush operation.\nShutdown GET,POST /ingester/shutdown # Legacy GET,POST /shutdown Flushes in-memory time series data from ingester to the long-term storage, and shuts down the ingester service. Notice that the other Cortex services are still running, and the operator (or any automation) is expected to terminate the process with a SIGINT / SIGTERM signal after the shutdown endpoint returns. In the meantime, /ready will not return 200. This endpoint will unregister the ingester from the ring even if -ingester.unregister-on-shutdown is disabled.\nThis API endpoint is usually used by scale down automations.\nIngesters ring status GET /ingester/ring # Legacy GET /ring Displays a web page with the ingesters hash ring status, including the state, healthy and last heartbeat time of each ingester.\nIngester tenants stats GET /ingester/all_user_stats Displays a web page with per-tenant statistics updated in realtime, including the total number of loaded blocks and active series from a specific ingester as well as the current ingestion rate (samples / sec).\nIngester mode GET,POST /ingester/mode Change ingester mode between ACTIVE or READONLY. READONLY ingester does not receive push requests and will only be called for query operations.\nThe endpoint accept query param mode or POST as application/x-www-form-urlencoded with mode type.\nQuerier / Query-frontend The following endpoints are exposed both by the querier and query-frontend.\nInstant query GET,POST \u003cprometheus-http-prefix\u003e/api/v1/query # Legacy GET,POST \u003clegacy-http-prefix\u003e/api/v1/query Prometheus-compatible instant query endpoint.\nFor more information, please check out the Prometheus instant query documentation.\nRequires authentication.\nRange query GET,POST \u003cprometheus-http-prefix\u003e/api/v1/query_range # Legacy GET,POST \u003clegacy-http-prefix\u003e/api/v1/query_range Prometheus-compatible range query endpoint. When the request is sent through the query-frontend, the query will be accelerated by query-frontend (results caching and execution parallelisation).\nFor more information, please check out the Prometheus range query documentation.\nRequires authentication.\nExemplar query GET,POST \u003cprometheus-http-prefix\u003e/api/v1/query_exemplars # Legacy GET,POST \u003clegacy-http-prefix\u003e/api/v1/query_exemplars Prometheus-compatible exemplar query endpoint.\nFor more information, please check out the Prometheus exemplar query documentation.\nRequires authentication.\nGet series by label matchers GET,POST \u003cprometheus-http-prefix\u003e/api/v1/series # Legacy GET,POST \u003clegacy-http-prefix\u003e/api/v1/series Find series by label matchers. Starting from release v1.18.0, Cortex by default honors the start and end request parameters and fetches series from either ingester, store gateway or both. The special case is that if start param is not specified, Cortex currently fetches series from data stored in the ingesters.\nFor more information, please check out the Prometheus series endpoint documentation.\nRequires authentication.\nGet label names GET,POST \u003cprometheus-http-prefix\u003e/api/v1/labels # Legacy GET,POST \u003clegacy-http-prefix\u003e/api/v1/labels Get label names of ingested series. Starting from release v1.18.0, Cortex by default honors the start and end request parameters and fetches label names from either ingester, store gateway or both.\nFor more information, please check out the Prometheus get label names documentation.\nRequires authentication.\nGet label values GET \u003cprometheus-http-prefix\u003e/api/v1/label/{name}/values # Legacy GET \u003clegacy-http-prefix\u003e/api/v1/label/{name}/values Get label values for a given label name. Starting from release v1.18.0, Cortex by default honors the start and end request parameters and fetches label values from either ingester, store gateway or both.\nFor more information, please check out the Prometheus get label values documentation.\nRequires authentication.\nGet metric metadata GET \u003cprometheus-http-prefix\u003e/api/v1/metadata # Legacy GET \u003clegacy-http-prefix\u003e/api/v1/metadata Prometheus-compatible metric metadata endpoint.\nFor more information, please check out the Prometheus metric metadata documentation.\nRequires authentication.\nRemote read POST \u003cprometheus-http-prefix\u003e/api/v1/read # Legacy POST \u003clegacy-http-prefix\u003e/api/v1/read Prometheus-compatible remote read endpoint.\nFor more information, please check out Prometheus Remote storage integrations.\nRequires authentication.\nBuild Information GET \u003cprometheus-http-prefix\u003e/api/v1/status/buildinfo # Legacy GET \u003clegacy-http-prefix\u003e/api/v1/status/buildinfo Prometheus-compatible build information endpoint.\nRequires authentication.\nQuerier Get tenant ingestion stats GET /api/v1/user_stats # Legacy GET \u003clegacy-http-prefix\u003e/user_stats Returns realtime ingestion rate, for the authenticated tenant, in JSON format.\nRequires authentication.\nRuler The ruler API endpoints require to configure a backend object storage to store the recording rules and alerts. The ruler API uses the concept of a “namespace” when creating rule groups. This is a stand in for the name of the rule file in Prometheus and rule groups must be named uniquely within a namespace.\nRuler ring status GET /ruler/ring # Legacy GET /ruler_ring Displays a web page with the ruler hash ring status, including the state, healthy and last heartbeat time of each ruler.\nRuler rules GET /ruler/rule_groups List all tenant rules. This endpoint is not part of ruler-API and is always available regardless of whether ruler-API is enabled or not. It should not be exposed to end users. This endpoint returns a YAML dictionary with all the rule groups for each tenant and 200 status code on success.\nList rules GET \u003cprometheus-http-prefix\u003e/api/v1/rules # Legacy GET \u003clegacy-http-prefix\u003e/api/v1/rules Prometheus-compatible rules endpoint to list alerting and recording rules that are currently loaded.\nFor more information, please check out the Prometheus rules documentation.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nList alerts GET \u003cprometheus-http-prefix\u003e/api/v1/alerts # Legacy GET \u003clegacy-http-prefix\u003e/api/v1/alerts Prometheus-compatible rules endpoint to list of all active alerts.\nFor more information, please check out the Prometheus alerts documentation.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nList rule groups GET /api/v1/rules # Legacy GET \u003clegacy-http-prefix\u003e/rules List all rules configured for the authenticated tenant. This endpoint returns a YAML dictionary with all the rule groups for each namespace and 200 status code on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nExample response --- \u003cnamespace1\u003e: - name: \u003cstring\u003e interval: \u003cduration;optional\u003e rules: - record: \u003cstring\u003e expr: \u003cstring\u003e - alert: \u003cstring\u003e expr: \u003cstring\u003e for: \u003cduration\u003e annotations: \u003cannotation_name\u003e: \u003cstring\u003e labels: \u003clabel_name\u003e: \u003cstring\u003e - name: \u003cstring\u003e interval: \u003cduration;optional\u003e rules: - record: \u003cstring\u003e expr: \u003cstring\u003e - alert: \u003cstring\u003e expr: \u003cstring\u003e for: \u003cduration\u003e annotations: \u003cannotation_name\u003e: \u003cstring\u003e labels: \u003clabel_name\u003e: \u003cstring\u003e \u003cnamespace2\u003e: - name: \u003cstring\u003e interval: \u003cduration;optional\u003e rules: - record: \u003cstring\u003e expr: \u003cstring\u003e - alert: \u003cstring\u003e expr: \u003cstring\u003e for: \u003cduration\u003e annotations: \u003cannotation_name\u003e: \u003cstring\u003e labels: \u003clabel_name\u003e: \u003cstring\u003e Get rule groups by namespace GET /api/v1/rules/{namespace} # Legacy GET \u003clegacy-http-prefix\u003e/rules/{namespace} Returns the rule groups defined for a given namespace.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nExample response name: \u003cstring\u003e interval: \u003cduration;optional\u003e rules: - record: \u003cstring\u003e expr: \u003cstring\u003e - alert: \u003cstring\u003e expr: \u003cstring\u003e for: \u003cduration\u003e annotations: \u003cannotation_name\u003e: \u003cstring\u003e labels: \u003clabel_name\u003e: \u003cstring\u003e Get rule group GET /api/v1/rules/{namespace}/{groupName} # Legacy GET \u003clegacy-http-prefix\u003e/rules/{namespace}/{groupName} Returns the rule group matching the request namespace and group name.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nSet rule group POST /api/v1/rules/{namespace} # Legacy POST \u003clegacy-http-prefix\u003e/rules/{namespace} Creates or updates a rule group. This endpoint expects a request with Content-Type: application/yaml header and the rules YAML definition in the request body, and returns 202 on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nExample request Request headers:\nContent-Type: application/yaml Request body:\nname: \u003cstring\u003e interval: \u003cduration;optional\u003e rules: - record: \u003cstring\u003e expr: \u003cstring\u003e - alert: \u003cstring\u003e expr: \u003cstring\u003e for: \u003cduration\u003e annotations: \u003cannotation_name\u003e: \u003cstring\u003e labels: \u003clabel_name\u003e: \u003cstring\u003e Delete rule group DELETE /api/v1/rules/{namespace}/{groupName} # Legacy DELETE \u003clegacy-http-prefix\u003e/rules/{namespace}/{groupName} Deletes a rule group by namespace and group name. This endpoints returns 202 on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nDelete namespace DELETE /api/v1/rules/{namespace} # Legacy DELETE \u003clegacy-http-prefix\u003e/rules/{namespace} Deletes all the rule groups in a namespace (including the namespace itself). This endpoint returns 202 on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nDelete tenant configuration POST /ruler/delete_tenant_config This deletes all rule groups for tenant, and returns 200 on success. Calling endpoint when no rule groups exist for user returns 200. Authentication is only to identify the tenant.\nThis is intended as internal API, and not to be exposed to users. This endpoint is enabled regardless of whether -experimental.ruler.enable-api is enabled or not.\nRequires authentication.\nAlertmanager Alertmanager status GET /multitenant_alertmanager/status # Legacy (microservices mode only) GET /status Displays a web page with the current status of the Alertmanager, including the Alertmanager cluster members.\nAlertmanager configs GET /multitenant_alertmanager/configs List all Alertmanager configurations. This endpoint is not part of alertmanager-API and is always available regardless of whether alertmanager-API is enabled or not. It should not be exposed to end users. This endpoint returns a YAML dictionary with all the Alertmanager configurations and 200 status code on success.\nAlertmanager ring status GET /multitenant_alertmanager/ring Displays a web page with the Alertmanager hash ring status, including the state, healthy and last heartbeat time of each Alertmanager instance.\nAlertmanager UI GET /\u003calertmanager-http-prefix\u003e # Legacy (microservices mode only) GET /\u003clegacy-http-prefix\u003e Displays the Alertmanager UI.\nRequires authentication.\nAlertmanager Delete Tenant Configuration POST /multitenant_alertmanager/delete_tenant_config This endpoint deletes configuration for a tenant identified by X-Scope-OrgID header. It is internal, available even if Alertmanager API is not enabled by using -experimental.alertmanager.enable-api. The endpoint returns a status code of 200 if the user’s configuration has been deleted, or it didn’t exist in the first place.\nRequires authentication.\nGet Alertmanager configuration GET /api/v1/alerts Get the current Alertmanager configuration for the authenticated tenant, reading it from the configured object storage.\nThis endpoint doesn’t accept any URL query parameter and returns 200 on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.alertmanager.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nSet Alertmanager configuration POST /api/v1/alerts Stores or updates the Alertmanager configuration for the authenticated tenant. The Alertmanager configuration is stored in the configured backend object storage.\nThis endpoint expects the Alertmanager YAML configuration in the request body and returns 201 on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.alertmanager.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nNote: When using curl send the request body from a file, ensure that you use the --data-binary flag instead of -d, --data, or --data-ascii. The latter options do not preserve carriage returns and newlines.\nExample request body template_files: default_template: | {{ define \"__alertmanager\" }}AlertManager{{ end }} {{ define \"__alertmanagerURL\" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver | urlquery }}{{ end }} alertmanager_config: | global: smtp_smarthost: 'localhost:25' smtp_from: 'youraddress@example.org' templates: - 'default_template' route: receiver: example-email receivers: - name: example-email email_configs: - to: 'youraddress@example.org' Delete Alertmanager configuration DELETE /api/v1/alerts Deletes the Alertmanager configuration for the authenticated tenant.\nThis endpoint doesn’t accept any URL query parameter and returns 200 on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.alertmanager.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nPurger The Purger service provides APIs for requesting deletion of tenants.\nTenant Delete Request POST /purger/delete_tenant Request deletion of ALL tenant data. Only works with blocks storage. Experimental.\nRequires authentication.\nTenant Delete Status GET /purger/delete_tenant_status Returns status of tenant deletion. Output format to be defined. Experimental.\nRequires authentication.\nStore-gateway Store-gateway ring status GET /store-gateway/ring Displays a web page with the store-gateway hash ring status, including the state, healthy and last heartbeat time of each store-gateway.\nCompactor Compactor ring status GET /compactor/ring Displays a web page with the compactor hash ring status, including the state, healthy and last heartbeat time of each compactor.\nConfigs API This service has been deprecated in favour of Ruler and Alertmanager API.\nThe configs API service provides an API-driven multi-tenant approach to handling various configuration files for Prometheus. The service hosts an API where users can read and write Prometheus rule files, Alertmanager configuration files, and Alertmanager templates to a database. Each tenant will have its own set of rule files, Alertmanager config, and templates.\nRequest / response schema The following schema is used both when retrieving the current configs from the API and when setting new configs via the API:\n{ \"id\": 99, \"rule_format_version\": \"2\", \"alertmanager_config\": \"\u003cstandard alertmanager.yaml config\u003e\", \"rules_files\": { \"rules.yaml\": \"\u003cstandard rules.yaml config\u003e\", \"rules2.yaml\": \"\u003cstandard rules.yaml config\u003e\" }, \"template_files\": { \"templates.tmpl\": \"\u003cstandard template file\u003e\", \"templates2.tmpl\": \"\u003cstandard template file\u003e\" } } id\nShould be incremented every time data is updated; Cortex will use the config with the highest number. rule_format_version\nAllows compatibility for tenants with config in Prometheus V1 format. Pass “1” or “2” according to which Prometheus version you want to match. alertmanager_config\nThe contents of the alertmanager config file should be as described here, encoded as a single string to fit within the overall JSON payload. config.rules_files\nThe contents of a rules file should be as described here, encoded as a single string to fit within the overall JSON payload. config.template_files\nThe contents of a template file should be as described here, encoded as a single string to fit within the overall JSON payload. These entries should match the templates entries in alertmanager_config. Example: template_files: myorg.tmpl: | {{ define \"__alertmanager\" }}AlertManager{{ end }} {{ define \"__alertmanagerURL\" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver | urlquery }}{{ end }} alertmanager_config: | templates: - 'myorg.tmpl' Get rule files GET /api/prom/configs/rules Get the current rule files for the authenticated tenant.\nRequires authentication.\nSet rule files POST /api/prom/configs/rules Replace the current rule files for the authenticated tenant.\nRequires authentication.\nGet template files GET /api/prom/configs/templates Get the current template files for the authenticated tenant.\nRequires authentication.\nSet template files POST /api/prom/configs/templates Replace the current template files for the authenticated tenant.\nRequires authentication.\nGet Alertmanager config file GET /api/prom/configs/alertmanager Get the current Alertmanager config for the authenticated tenant.\nRequires authentication.\nSet Alertmanager config file POST /api/prom/configs/alertmanager Replace the current Alertmanager config for the authenticated tenant.\nRequires authentication.\nValidate Alertmanager config file POST /api/prom/configs/alertmanager/validate Validate the Alertmanager config in the request body. The request body is expected to contain only the Alertmanager YAML config.\nDeactivate configs DELETE /api/prom/configs/deactivate Disable configs for the authenticated tenant. Please be aware that setting a new config will effectively “re-enable” the Rules and Alertmanager configuration for the tenant.\nRequires authentication.\nRestore configs POST /api/prom/configs/restore Re-enable configs for the authenticated tenant, after being previously deactivated.\nRequires authentication.\n","categories":"","description":"","excerpt":"Cortex exposes an HTTP API for pushing and querying time series data, …","ref":"/docs/api/","tags":"","title":"HTTP API"},{"body":"The Cortex blocks storage engine stores series in TSDB blocks uploaded in the storage bucket. This makes very easy to migrate the storage from Thanos and/or Prometheus to Cortex, when running the blocks storage.\nCortex blocks storage requirements The Cortex blocks storage has few requirements that should be considered when migrating TSDB blocks from Thanos / Prometheus to Cortex:\nThe blocks in the bucket should be located at bucket://\u003ctenant-id\u003e/\nCortex isolates blocks on a per-tenant basis in the bucket and, for this reason, each tenant blocks should be uploaded to a different location in the bucket. The bucket prefix, where a specific tenant blocks should be uploaded, is /\u003ctenant-id\u003e/; if Cortex is running with auth disabled (no multi-tenancy) then the \u003ctenant-id\u003e to use is fake. Remove Thanos external labels and inject __org_id__ into each block’s meta.json\nEvery block has a little metadata file named meta.json. Thanos stores external labels at thanos \u003e labels, which should be all removed when migrating to Cortex, while the \"__org_id__\": \"\u003ctenant-id\u003e\" added. How to migrate the storage Upload TSDB blocks to Cortex bucket TSDB blocks stored in Prometheus local disk or Thanos bucket should be copied/uploaded to the Cortex bucket at the location bucket://\u003ctenant-id\u003e/ (when Cortex is running with auth disabled then \u003ctenant-id\u003e must be fake).\nMigrate block metadata (meta.json) to Cortex For each block copied/uploaded to the Cortex bucket, there are a few changes required to the meta.json.\nAutomatically migrate metadata using the thanosconvert tool thanosconvert can iterate over a Cortex bucket and make sure that each meta.json has the correct thanos \u003e labels layout.\n⚠ Warning ⚠ thanosconvert will modify files in the bucket you specify. It’s recommended that you have backups or enable object versioning before running this tool.\nTo run thanosconvert, you need to provide it with the bucket configuration in the same format as the blocks storage bucket configuration.\n# bucket-config.yaml backend: s3 s3: endpoint: s3.us-east-1.amazonaws.com bucket_name: my-cortex-bucket You can run thanosconvert directly using Go:\ngo install github.com/cortexproject/cortex/cmd/thanosconvert thanosconvert Or use the provided docker image:\ndocker run quay.io/cortexproject/thanosconvert You can run the tool in dry-run mode first to find out what which blocks it will migrate:\nthanosconvert -config ./bucket-config.yaml -dry-run Once you’re happy with the results, you can run without dry run to migrate blocks:\nthanosconvert -config ./bucket-config.yaml You can cancel a conversion in progress (with Ctrl+C) and rerun thanosconvert. It won’t change any blocks which have been written by Cortex or already converted from Thanos, so you can run thanosconvert multiple times.\nMigrate metadata manually If you need to migrate the block metadata manually, you need to:\nDownload the meta.json to the local filesystem Decode the JSON Manipulate the data structure (see below) Re-encode the JSON Re-upload it to the bucket (overwriting the previous version of the meta.json file) The meta.json should be manipulated in order to ensure:\nIt contains the thanos root-level entry The thanos \u003e labels do not contain any Thanos-specific external label The thanos \u003e labels contain the Cortex-specific external label \"__org_id__\": \"\u003ctenant-id\u003e\" When migrating from Thanos When migrating from Thanos, the easiest approach would be keep the existing thanos root-level entry as is, except:\nCompletely remove the content of thanos \u003e labels Add \"__org_id__\": \"\u003ctenant-id\u003e\" to thanos \u003e labels For example, when migrating a block from Thanos for the tenant user-1, the thanos root-level property within the meta.json file will look like:\n{ \"thanos\": { \"labels\": { \"__org_id__\": \"user-1\" }, \"downsample\": { \"resolution\": 0 }, \"source\": \"compactor\" } } Right now Cortex doesn’t support downsampling so Thanos downsampled blocks are not supported. Downsampled blocks will be simply skipped by the thanosconvert tool.\nIf downsampled blocks are uploaded to the Cortex bucket, they cannot be queried so please exclude them when migrating TSDB blocks.\nWhen migrating from Prometheus When migrating from Prometheus, the meta.json file will not contain any thanos root-level entry and, for this reason, it would need to be generated:\nCreate the thanos root-level entry (see below) Add \"__org_id__\": \"\u003ctenant-id\u003e\" to thanos \u003e labels For example, when migrating a block from Prometheus for the tenant user-1, the thanos root-level property within the meta.json file should be as follow:\n{ \"thanos\": { \"labels\": { \"__org_id__\": \"user-1\" }, \"downsample\": { \"resolution\": 0 }, \"source\": \"compactor\" } } ","categories":"","description":"","excerpt":"The Cortex blocks storage engine stores series in TSDB blocks uploaded …","ref":"/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/","tags":"","title":"Migrate the storage from Thanos and Prometheus"},{"body":"","categories":"","description":"","excerpt":"","ref":"/docs/operations/","tags":"","title":"Operating Cortex"},{"body":"","categories":"","description":"","excerpt":"","ref":"/docs/case-studies/","tags":"","title":"Case Studies"},{"body":"","categories":"","description":"","excerpt":"","ref":"/docs/proposals/","tags":"","title":"Proposals"},{"body":"Context Cortex Alertmanager notification setup follows mostly the syntax of Prometheus Alertmanager since it is based on the same codebase. The following is a description on how to load the configuration setup so that Alertmanager can use it for notification when an alert event happens.\nConfiguring the Cortex Alertmanager storage backend With the introduction of Cortex 1.8, the storage backend config option shifted to the new pattern #3888. You can find the new configuration here\nNote that when using -alertmanager.sharding-enabled=true, the following storage backends are not supported: local, configdb.\nWhen using the new configuration pattern, it is important that any of the old configuration pattern flags are unset (-alertmanager.storage), as well as -\u003cprefix\u003e.configs.url. This is because the old pattern still takes precedence over the new one. The old configuration pattern (-alertmanager.storage) is marked as deprecated and will be removed by Cortex version 1.11. However, this change doesn’t apply to -alertmanager.storage.path and -alertmanager.storage.retention.\nCortex Alertmanager configuration Cortex Alertmanager can be uploaded via Cortex Set Alertmanager configuration API or using Cortex Tools.\nFollow the instructions at the cortextool link above to download or update to the latest version of the tool.\nTo obtain the full help of how to use cortextool for all commands and flags, use cortextool --help-long.\nThe following example shows the steps to upload the configuration to Cortex Alertmanager using cortextool.\n1. Create the Alertmanager configuration YAML file. The following is amconfig.yml, an example of a configuration for Cortex Alertmanager to send notifications via email:\nglobal: # The smarthost and SMTP sender used for mail notifications. smtp_smarthost: 'localhost:25' smtp_from: 'alertmanager@example.org' smtp_auth_username: 'alertmanager' smtp_auth_password: 'password' route: # A default receiver. receiver: send-email receivers: - name: send-email email_configs: - to: 'someone@localhost' Example on how to set up Slack to support receiving Alertmanager notifications.\n2. Upload the Alertmanager configuration In this example, Cortex Alertmanager is set to be available via localhost on port 8095 with user/org = 100.\nTo upload the above configuration .yml file with --key to be your Basic Authentication or API key:\ncortextool alertmanager load ./amconfig.yml \\ --address=http://localhost:8095 \\ --id=100 \\ --key=\u003cyourKey\u003e If there is no error reported, the upload is successful.\nTo upload the configuration for Cortex Alertmanager using Cortex API and curl - see Cortex Set Alertmanager configuration API.\n3. Ensure the configuration has been uploaded successfully cortextool alertmanager get \\ --address=http://localhost:8095 \\ --id=100 \\ --key=\u003cyourKey\u003e ","categories":"","description":"","excerpt":"Context Cortex Alertmanager notification setup follows mostly the …","ref":"/docs/guides/alertmanager-configuration/","tags":"","title":"Configuring Notification using Cortex Alertmanager"},{"body":"All Cortex components take the tenant ID from a header X-Scope-OrgID on each request. A tenant (also called “user” or “org”) is the owner of a set of series written to and queried from Cortex. All Cortex components trust this value completely: if you need to protect your Cortex installation from accidental or malicious calls, then you must add an additional layer of protection.\nTypically, this means you run Cortex behind a reverse proxy, and you must ensure that all callers, both machines sending data over the remote_write interface and humans sending queries from GUIs, supply credentials which identify them and confirm they are authorised. When configuring the remote_write API in Prometheus, the user and password fields of HTTP Basic auth, or Bearer token, can be used to convey the tenant ID and/or credentials. See the Cortex-Tenant section below for one way to solve this.\nIn trusted environments, Prometheus can send the X-Scope-OrgID header itself by configuring the headers field in its remote_write configuration:\nremote_write: - url: http://\u003ccortex\u003e/prometheus/api/v1/push headers: X-Scope-OrgID: \u003corg\u003e To disable the multi-tenant functionality, you can pass the argument -auth.enabled=false to every Cortex component, which will set the OrgID to the string fake for every request.\nNote that the tenant ID that is used to write the series to the datastore should be the same as the one you use to query the data. If they don’t match, you won’t see any data. As of now, you can’t see series from other tenants.\nFor more information regarding the tenant ID limits, refer to: Tenant ID limitations\nCortex-Tenant One way to add X-Scope-OrgID to Prometheus requests is to use a cortex-tenant proxy which is able to extract the tenant ID from Prometheus labels.\nIt can be placed between Prometheus and Cortex and will search for a predefined label and use its value as X-Scope-OrgID header when proxying the timeseries to Cortex.\nThis can help to run Cortex in a trusted environment where you want to separate your metrics into distinct namespaces by some criteria (e.g. teams, applications, etc.).\nBe advised that cortex-tenant is a third-party community project and it’s not maintained by the Cortex team.\n","categories":"","description":"","excerpt":"All Cortex components take the tenant ID from a header X-Scope-OrgID …","ref":"/docs/guides/auth/","tags":"","title":"Authentication and Authorisation"},{"body":"This doc is likely out of date. It should be updated for blocks storage.\nYou will want to estimate how many nodes are required, how many of each component to run, and how much storage space will be required. In practice, these will vary greatly depending on the metrics being sent to Cortex.\nSome key parameters are:\nThe number of active series. If you have Prometheus already, you can query prometheus_tsdb_head_series to see this number. Sampling rate, e.g. a new sample for each series every minute (the default Prometheus scrape_interval). Multiply this by the number of active series to get the total rate at which samples will arrive at Cortex. The rate at which series are added and removed. This can be very high if you monitor objects that come and go - for example, if you run thousands of batch jobs lasting a minute or so and capture metrics with a unique ID for each one. Read how to analyse this on Prometheus. How compressible the time-series data are. If a metric stays at the same value constantly, then Cortex can compress it very well, so 12 hours of data sampled every 15 seconds would be around 2KB. On the other hand, if the value jumps around a lot, it might take 10KB. There are not currently any tools available to analyse this. How long you want to retain data for, e.g. 1 month or 2 years. Other parameters which can become important if you have particularly high values:\nNumber of different series under one metric name. Number of labels per series. Rate and complexity of queries. Now, some rules of thumb:\nEach million series in an ingester takes 15GB of RAM. The total number of series in ingesters is the number of active series times the replication factor. This is with the default of 12-hour chunks - RAM required will reduce if you set -ingester.max-chunk-age lower (trading off more back-end database I/O). There are some additional considerations for planning for ingester memory usage. Memory increases during write-ahead log (WAL) replay, See Prometheus issue #6934. If you do not have enough memory for WAL replay, the ingester will not be able to restart successfully without intervention. Memory temporarily increases during resharding since timeseries are temporarily on both the new and old ingesters. This means you should scale up the number of ingesters before memory utilization is too high, otherwise you will not have the headroom to account for the temporary increase. Each million series (including churn) consumes 15GB of chunk storage and 4GB of index, per day (so multiply by the retention period). The distributors’ CPU utilization depends on the specific Cortex cluster setup, while they don’t need much RAM. Typically, distributors are capable of processing between 20,000 and 100,000 samples/sec with 1 CPU core. It’s also highly recommended to configure Prometheus max_samples_per_send to 1,000 samples, in order to reduce the distributors’ CPU utilization given the same total samples/sec throughput. If you turn on compression between distributors and ingesters (for example, to save on inter-zone bandwidth charges at AWS/GCP), they will use significantly more CPU (approx. 100% more for distributor and 50% more for ingester).\n","categories":"","description":"","excerpt":"This doc is likely out of date. It should be updated for blocks …","ref":"/docs/guides/capacity-planning/","tags":"","title":"Capacity Planning"},{"body":"Context One option to scale the ruler is by scaling it horizontally. However, with multiple ruler instances running, they will need to coordinate to determine which instance will evaluate which rule. Similar to the ingesters, the rulers establish a hash ring to divide up the responsibilities of evaluating rules.\nConfig In order to enable sharding in the ruler, the following flag needs to be set:\n-ruler.enable-sharding=true In addition, the ruler requires its own ring to be configured, for instance:\n-ruler.ring.consul.hostname=consul.dev.svc.cluster.local:8500 The only configuration that is required is to enable sharding and configure a key-value store. From there, the rulers will shard and handle the division of rules automatically.\nUnlike ingesters, rulers do not hand over responsibility: all rules are re-sharded randomly every time a ruler is added to or removed from the ring.\nRuler Storage The ruler supports six kinds of storage (configdb, Azure, GCS, S3, Swift, local). Most kinds of storage work with the sharded ruler configuration in an obvious way, i.e. configure all rulers to use the same backend.\nThe local implementation reads Prometheus recording rules off the local filesystem. This is a read-only backend that does not support the creation and deletion of rules through the API. Despite the fact that it reads the local filesystem, this method can still be used in a sharded ruler configuration if the operator takes care to load the same rules to every ruler. For instance, this could be accomplished by mounting a Kubernetes ConfigMap onto every ruler pod.\nA typical local config may look something like:\n-ruler-storage.backend=local -ruler-storage.local.directory=/tmp/cortex/rules With the above configuration, the ruler would expect the following layout:\n/tmp/cortex/rules/\u003ctenant id\u003e/rules1.yaml /rules2.yaml Yaml files are expected to be in the Prometheus format.\n","categories":"","description":"","excerpt":"Context One option to scale the ruler is by scaling it horizontally. …","ref":"/docs/guides/ruler-sharding/","tags":"","title":"Config for horizontally scaling the Ruler"},{"body":"Context You can have more than a single Prometheus monitoring and ingesting the same metrics for redundancy. Cortex already does replication for redundancy, and it doesn’t make sense to ingest the same data twice. So in Cortex, we made sure we can dedupe the data we receive from HA Pairs of Prometheus. We do this via the following:\nAssume that there are two teams, each running their own Prometheus, monitoring different services. Let’s call the Prometheus T1 and T2. Now, if the teams are running HA pairs, let’s call the individual Prometheus, T1.a, T1.b, and T2.a and T2.b.\nIn Cortex, we make sure we only ingest from one of T1.a and T1.b, and only from one of T2.a and T2.b. We do this by electing a leader replica for each cluster of Prometheus. For example, in the case of T1, let it be T1.a. As long as T1.a is the leader, we drop the samples sent by T1.b. And if Cortex sees no new samples from T1.a for a short period (30s by default), it’ll switch the leader to be T1.b.\nThis means if T1.a goes down for a few minutes, Cortex’s HA sample handling will have switched and elected T1.b as the leader. This failover timeout is what enables us to only accept samples from a single replica at a time, but ensure we don’t drop too much data in case of issues. Note that with the default scrape period of 15s, and the default timeouts in Cortex, in most cases, you’ll only lose a single scrape of data in the case of a leader election failover. For any rate queries, the rate window should be at least 4x the scrape period to account for any of these failover scenarios, for example, with the default scrape period of 15s, then you should calculate rates over at least 1m periods.\nNow we do the same leader election process for T2.\nConfig Client Side So for Cortex to achieve this, we need 2 identifiers for each process, one identifier for the cluster (T1 or T2, etc.) and one identifier to identify the replica in the cluster (a or b). The easiest way to do this is by setting external labels; the default labels are cluster and __replica__. For example:\ncluster: prom-team1 __replica__: replica1 (or pod-name) and\ncluster: prom-team1 __replica__: replica2 Note: These are external labels and have nothing to do with remote_write config.\nThese two label names are configurable per-tenant within Cortex and should be set to something sensible. For example, the cluster label is already used by some workloads, and you should set the label to be something else that uniquely identifies the cluster. Good examples for this label-name would be team, cluster, prometheus, etc.\nThe replica label should be set so that the value for each prometheus is unique in that cluster. Note: Cortex drops this label when ingesting data but preserves the cluster label. This way, your timeseries won’t change when replicas change.\nServer Side The minimal configuration requires:\nEnabling the HA tracker via -distributor.ha-tracker.enable=true CLI flag (or its YAML config option) Configuring the KV store for the ring (See: Ring/HA Tracker Store). Only Consul and etcd are currently supported. Multi should be used for migration purposes only. Setting the limits configuration to accept samples via -distributor.ha-tracker.enable-for-all-users (or its YAML config option). The following configuration snippet shows an example of the HA tracker config via YAML config file:\nlimits: ... accept_ha_samples: true ... distributor: ... ha_tracker: enable_ha_tracker: true ... kvstore: [store: \u003cstring\u003e | default = \"consul\"] [consul | etcd: \u003cconfig\u003e] ... ... For further configuration file documentation, see the distributor section and Ring/HA Tracker Store.\nFor flag configuration, see the distributor flags having ha-tracker in them.\nRemote Read If you plan to use remote_read, you can’t have the __replica__ label in the external section. Instead, you will need to add it only on the remote_write section of your prometheus.yml.\nglobal: external_labels: cluster: prom-team1 remote_write: - url: https://cortex/api/v1/push write_relabel_configs: - target_label: __replica__ replacement: 1 and\nglobal: external_labels: cluster: prom-team1 remote_write: - url: https://cortex/api/v1/push write_relabel_configs: - target_label: __replica__ replacement: replica2 When Prometheus is executing remote read queries, it will add the external labels to the query. In this case, if it asks for the __replica__ label, Cortex will not return any data.\nTherefore, the __replica__ label should only be added for remote write.\nAccept multiple HA pairs in single request Let’s assume there are two teams (T1 and T2), and each team operates two Prometheus for the HA (T1.a, T1.b for T1 and T2.a, T2.b for T2). They want to operate another Prometheus, receiving whole Prometheus requests and sending write request to the Distributor.\nThe write request flow is as follows: T1.a, T1.b, T2.a, T2.b -\u003e Prometheus -\u003e Distributor which means the Distributor’s incoming write request contains time series of T1.a, T1.b, T2.a, and T2.b. In other words, there are two HA pairs in a single write request, and the expected push result is to accept each Prometheus leader replicas (example: T1.a, T2.b for each team).\nConfig Client side The client setting is the same as a single HA pair. For example:\nFor T1.a\ncluster: prom-team1 __replica__: replica1 (or pod-name) For T1.b\ncluster: prom-team1 __replica__: replica2 (or pod-name) For T2.a\ncluster: prom-team2 __replica__: replica1 (or pod-name) For T2.b\ncluster: prom-team2 __replica__: replica2 (or pod-name) Server side One additional setting is needed to accept multiple HA pairs; it is enabled via --experimental.distributor.ha-tracker.mixed-ha-samples=true (or its YAML config option).\nThe following configuration snippet shows an example of accepting multiple HA pairs config via the YAML config file:\nlimits: ... accept_ha_samples: true accept_mixed_ha_samples: true ... distributor: ... ha_tracker: enable_ha_tracker: true ... kvstore: [ store: \u003cstring\u003e | default = \"consul\" ] [ consul | etcd: \u003cconfig\u003e ] ... ... For further configuration file documentation, see the limits section.\n","categories":"","description":"","excerpt":"Context You can have more than a single Prometheus monitoring and …","ref":"/docs/guides/ha-pair-handling/","tags":"","title":"Config for sending HA Pairs data to Cortex"},{"body":"Welcome! We’re excited that you’re interested in contributing. Below are some basic guidelines.\nWorkflow Cortex follows a standard GitHub pull request workflow. If you’re unfamiliar with this workflow, read the very helpful Understanding the GitHub flow guide from GitHub.\nYou are welcome to create draft PRs at any stage of readiness - this can be helpful to ask for assistance or to develop an idea. But before a piece of work is finished it should:\nBe organized into one or more commits, each of which has a commit message that describes all changes made in that commit (‘why’ more than ‘what’ - we can read the diffs to see the code that changed). Each commit should build towards the whole - don’t leave in back-tracks and mistakes that you later corrected. Have unit and/or integration tests for new functionality or tests that would have caught the bug being fixed. Include a CHANGELOG message if users of Cortex need to hear about what you did. If you have made any changes to flags or config, run make doc and commit the changed files to update the config file documentation. Formatting Cortex projects uses goimports tool (go get golang.org/x/tools/cmd/goimports to install) to format the Go files, and sort imports. We use goimports with -local github.com/cortexproject/cortex parameter, to put Cortex internal imports into a separate group. We try to keep imports sorted into three groups: imports from standard library, imports of 3rd party packages and internal Cortex imports. Goimports will fix the order, but will keep existing newlines between imports in the groups. We try to avoid extra newlines like that.\nYou’re using an IDE you may find useful the following settings for the Cortex project:\nVSCode Developer Certificates of Origin (DCOs) Before submitting your work in a pull request, make sure that all commits are signed off with a Developer Certificate of Origin (DCO). Here’s an example:\ngit commit -s -m \"Here is my signed commit\" You can find further instructions here.\nBuilding Cortex To build:\nmake (By default, the build runs in a Docker container, using an image built with all the tools required. The source code is mounted from where you run make into the build container as a Docker volume.)\nTo run the unit tests suite:\ngo test ./... To run the integration tests suite please see “How integration tests work”.\nDependency management We use Go modules to manage dependencies on external packages. This requires a working Go environment with version 1.11 or greater, git and bzr installed.\nTo add or update a new dependency, use the go get command:\n# Pick the latest tagged release. go get example.com/some/module/pkg # Pick a specific version. go get example.com/some/module/pkg@vX.Y.Z Tidy up the go.mod and go.sum files:\ngo mod tidy go mod vendor git add go.mod go.sum vendor git commit You have to commit the changes to go.mod and go.sum before submitting the pull request.\nDesign patterns and Code conventions Please see the dedicated “Design patterns and Code conventions” page.\nDocumentation The Cortex documentation is compiled into a website published at cortexmetrics.io. Please see “How to run the website locally” for instructions.\nNote: if you attempt to view pages on Github, it’s likely that you might find broken links or pages. That is expected and should not be addressed unless it is causing issues with the site that occur as part of the build.\n","categories":"","description":"","excerpt":"Welcome! We’re excited that you’re interested in contributing. Below …","ref":"/docs/contributing/","tags":"","title":"Contributing"},{"body":"Cortex supports data encryption at rest for some storage backends.\nS3 The Cortex S3 client supports the following server-side encryption (SSE) modes:\nSSE-S3 SSE-KMS Blocks storage The blocks storage S3 server-side encryption can be configured as follows.\ns3_sse_config The s3_sse_config configures the S3 server-side encryption.\nsse: # Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3. # CLI flag: -\u003cprefix\u003e.s3.sse.type [type: \u003cstring\u003e | default = \"\"] # KMS Key ID used to encrypt objects in S3 # CLI flag: -\u003cprefix\u003e.s3.sse.kms-key-id [kms_key_id: \u003cstring\u003e | default = \"\"] # KMS Encryption Context used for object encryption. It expects JSON formatted # string. # CLI flag: -\u003cprefix\u003e.s3.sse.kms-encryption-context [kms_encryption_context: \u003cstring\u003e | default = \"\"] Ruler The ruler S3 server-side encryption can be configured similarly to the blocks storage. The per-tenant overrides are supported when using the storage backend configurable the -ruler-storage. flag prefix (or their respective YAML config options).\nAlertmanager The alertmanager S3 server-side encryption can be configured similarly to the blocks storage. The per-tenant overrides are supported when using the storage backend configurable the -alertmanager-storage. flag prefix (or their respective YAML config options).\nPer-tenant config overrides The S3 client used by the blocks storage, ruler, and alertmanager supports S3 SSE config overrides on a per-tenant basis, using the runtime configuration file. The following settings can be overridden for each tenant:\ns3_sse_type\nS3 server-side encryption type. It must be set to enable the SSE config override for a given tenant. s3_sse_kms_key_id\nS3 server-side encryption KMS Key ID. Ignored if the SSE type override is not set or the type is not SSE-KMS. s3_sse_kms_encryption_context\nS3 server-side encryption KMS encryption context. If unset and the key ID override is set, the encryption context will not be provided to S3. Ignored if the SSE type override is not set or the type is not SSE-KMS. Other storages Other storage backends may support encryption at rest, configuring it directly at the storage level.\n","categories":"","description":"","excerpt":"Cortex supports data encryption at rest for some storage backends.\nS3 …","ref":"/docs/guides/encryption-at-rest/","tags":"","title":"Encryption at Rest"},{"body":"New maintainers are proposed by an existing maintainer and are elected by majority vote. Once the vote passed, the following steps should be done to add a new member to the maintainers team:\nSubmit a PR to add the new member to MAINTAINERS Invite to GitHub organization Invite to cortex-team group Invite to Quay.io repository Invite to Docker Hub organization Submit a PR to CNCF maintainer list for Cortex Invite to CNCF cncf-cortex-maintainers mailing list (via CNCF Service Desk) Invite to Cortex Project Public Calendar Invite to maintainer’s Google Docs Add to the Google Analytics property used for the website statistics Invite to credentials vault ","categories":"","description":"","excerpt":"New maintainers are proposed by an existing maintainer and are elected …","ref":"/docs/contributing/how-to-add-a-maintainer/","tags":"","title":"How to add a maintainer"},{"body":"Cortex ingesters are semi-stateful. A running ingester holds several hours of time series data in memory before they’re flushed to the long-term storage. When an ingester shuts down because of a rolling update or maintenance, the in-memory data must not be discarded in order to avoid any data loss.\nThe Cortex blocks storage requires ingesters to run with a persistent disk where the TSDB WAL and blocks are stored (eg. a StatefulSet when deployed on Kubernetes).\nDuring a rolling update, the leaving ingester closes the open TSDBs, synchronizes the data to disk (fsync), and releases the disk resources. The new ingester, which is expected to reuse the same disk as the leaving one, will replay the TSDB WAL on startup to load back into memory the time series that have not been compacted into a block yet.\n","categories":"","description":"","excerpt":"Cortex ingesters are semi-stateful. A running ingester holds several …","ref":"/docs/guides/ingesters-rolling-updates/","tags":"","title":"Ingesters rolling updates"},{"body":"This guide explains how to scale up and down ingesters.\nIf you’re looking for how to run ingesters rolling updates, please refer to the dedicated guide._\nScaling up Adding more ingesters to a Cortex cluster is considered a safe operation. When a new ingester starts, it will register to the hash ring and the distributors will reshard received series accordingly. Ingesters that were previously receiving those series will see data stop arriving and will consider those series “idle”.\nIf you run with -distributor.shard-by-all-labels=false (the default), before adding a second ingester, you have to wait until data has migrated from idle series to the back-end store; otherwise, you will see gaps in queries. This will happen after the next “head compaction” (typically every 2 hours). If you have set -querier.query-store-after, then that is also a minimum time you have to wait before adding a second ingester.\nIf you run with -distributor.shard-by-all-labels=true, no special care is required to take when scaling up ingesters.\nScaling down A running ingester holds several hours of time series data in memory before they’re flushed to the long-term storage. When an ingester shuts down because of a scale down operation, the in-memory data must not be discarded in order to avoid any data loss.\nIngesters don’t flush series to blocks at shutdown by default. However, Cortex ingesters expose an API endpoint /shutdown that can be called to flush series to blocks and upload blocks to the long-term storage before the ingester terminates.\nEven if ingester blocks are compacted and shipped to the storage at shutdown, it takes some time for queriers and store-gateways to discover the newly uploaded blocks. This is due to the fact that the blocks storage runs a periodic scanning of the storage bucket to discover blocks. If two or more ingesters are scaled down in a short period of time, queriers may miss some data at query time due to series that were stored in the terminated ingesters but their blocks haven’t been discovered yet.\nThe ingesters scale down is deemed an infrequent operation and no automation is currently provided. However, if you need to scale down ingesters, please be aware of the following:\nConfigure queriers and rulers to always query the storage -querier.query-store-after=0s Frequently scan the storage bucket -blocks-storage.bucket-store.sync-interval=5m -compactor.cleanup-interval=5m Lower bucket scanning cache TTLs -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl=1m -blocks-storage.bucket-store.metadata-cache.tenant-blocks-list-ttl=1m -blocks-storage.bucket-store.metadata-cache.metafile-doesnt-exist-ttl=1m Ingesters should be scaled down one by one: Call /shutdown endpoint on the ingester to shutdown Wait until the HTTP call returns successfully or “finished flushing and shipping TSDB blocks” is logged Terminate the ingester process (the /shutdown will not do it) Before proceeding to the next ingester, wait 2x the maximum between -blocks-storage.bucket-store.sync-interval and -compactor.cleanup-interval ","categories":"","description":"","excerpt":"This guide explains how to scale up and down ingesters.\nIf you’re …","ref":"/docs/guides/ingesters-scaling-up-and-down/","tags":"","title":"Ingesters scaling up and down"},{"body":"Since Cortex is a multi-tenant system, it supports applying limits to each tenant to prevent any single one from using too many resources. In order to help operators understand how close to their limits tenants are, the overrides-exporter module can expose limits as Prometheus metrics.\nContext To update configuration without restarting, Cortex allows operators to supply a runtime_config file that will be periodically reloaded. This file can be specified under the runtime_config section of the main configuration file or using the -runtime-config.file command-line flag. This file is used to apply tenant-specific limits.\nExample The overrides-exporter is not enabled by default; it must be explicitly enabled. We recommend only running a single instance of it in your cluster due to the cardinality of the metrics emitted.\nWith a runtime.yaml file given below:\n# file: runtime.yaml # In this example, we're overriding ingestion limits for a single tenant. overrides: \"user1\": ingestion_burst_size: 350000 ingestion_rate: 350000 max_global_series_per_metric: 300000 max_global_series_per_user: 300000 max_series_per_metric: 0 max_series_per_user: 0 max_samples_per_query: 100000 max_series_per_query: 100000 The overrides-exporter is configured to run as follows:\ncortex -target overrides-exporter -runtime-config.file runtime.yaml -server.http-listen-port=8080 After the overrides-exporter starts, you can use curl to inspect the tenant overrides.\ncurl -s http://localhost:8080/metrics | grep cortex_overrides # HELP cortex_overrides Resource limit overrides applied to tenants # TYPE cortex_overrides gauge cortex_overrides{limit_name=\"ingestion_burst_size\",user=\"user1\"} 350000 cortex_overrides{limit_name=\"ingestion_rate\",user=\"user1\"} 350000 cortex_overrides{limit_name=\"max_global_series_per_metric\",user=\"user1\"} 300000 cortex_overrides{limit_name=\"max_global_series_per_user\",user=\"user1\"} 300000 cortex_overrides{limit_name=\"max_local_series_per_metric\",user=\"user1\"} 0 cortex_overrides{limit_name=\"max_local_series_per_user\",user=\"user1\"} 0 cortex_overrides{limit_name=\"max_samples_per_query\",user=\"user1\"} 100000 cortex_overrides{limit_name=\"max_series_per_query\",user=\"user1\"} 100000 With these metrics, you can set up alerts to know when tenants are close to hitting their limits before they exceed them.\n","categories":"","description":"","excerpt":"Since Cortex is a multi-tenant system, it supports applying limits to …","ref":"/docs/guides/overrides-exporter/","tags":"","title":"Overrides Exporter"},{"body":"This document highlights some ideas for major features we’d like to implement in the near future. To get a more complete overview of planned features and current work, see the issue tracker. Note that these are not ordered by priority.\nHelm charts and other packaging We have a helm chart but it needs work before it can be effectively utilised by different backends. We also don’t provide an official set of dashboards and alerts to our users yet. This is one of the most requested features and something we will tackle in the immediate future. We also plan on publishing debs, rpms along with guides on how to run Cortex on bare-metal.\nAuth Gateway Cortex server has a simple authentication mechanism (X-Scope-OrgId) but users can’t use the multitenancy features out of the box without complicated proxy configuration. It’s hard to support all the different authentication mechanisms used by different companies but plan to have a simple but opinionated auth-gateway that provides value out of the box. The configuration could be as simple as:\ntenants: - name: infra-team password: basic-auth-password - name: api-team password: basic-auth-password2 Billing and Usage analytics We have all the metrics to track how many series, samples and queries each tenant is sending but don’t have dashboards that help with this. We plan to have dashboards and UIs that will help operators monitor and control each tenants usage out of the box.\nDownsampling Downsampling means storing fewer samples, e.g. one per minute instead of one every 15 seconds. This makes queries over long periods more efficient. It can reduce storage space slightly if the full-detail data is discarded.\nPer-metric retention Cortex blocks storage supports deleting all data for a tenant after a time period (e.g. 3 months, 1 year), but we would also like to have custom retention for subsets of metrics (e.g. delete server metrics but retain business metrics).\nExemplar support Exemplars let you link metric samples to other data, such as distributed tracing. As of early 2021 Prometheus will collect exemplars and send them via remote write, but Cortex needs to be extended to handle them.\nScalability Scalability has always been a focus for the project, but there is a lot more work to be done. We can now scale to 100s of Millions of active series but 1 Billion active series is still an unknown.\n","categories":"","description":"","excerpt":"This document highlights some ideas for major features we’d like to …","ref":"/docs/roadmap/","tags":"","title":"Roadmap"},{"body":"Cortex is a distributed system with significant traffic between its services. To allow for secure communication, Cortex supports TLS between all its components. This guide describes the process of setting up TLS.\nGeneration of certs to configure TLS The first step to securing inter-service communication in Cortex with TLS is generating certificates. A Certifying Authority (CA) will be used for this purpose, which should be private to the organization, as any certificates signed by this CA will have permissions to communicate with the cluster.\nWe will use the following script to generate self-signed certs for the cluster:\n# keys openssl genrsa -out root.key openssl genrsa -out client.key openssl genrsa -out server.key # root cert / certifying authority openssl req -x509 -new -nodes -key root.key -subj \"/C=US/ST=KY/O=Org/CN=root\" -sha256 -days 100000 -out root.crt # csrs - certificate signing requests openssl req -new -sha256 -key client.key -subj \"/C=US/ST=KY/O=Org/CN=client\" -out client.csr openssl req -new -sha256 -key server.key -subj \"/C=US/ST=KY/O=Org/CN=localhost\" -out server.csr # certificates openssl x509 -req -in client.csr -CA root.crt -CAkey root.key -CAcreateserial -out client.crt -days 100000 -sha256 openssl x509 -req -in server.csr -CA root.crt -CAkey root.key -CAcreateserial -out server.crt -days 100000 -sha256 Note that the above script generates certificates that are valid for 100,000 days. This can be changed by adjusting the -days option in the above commands. It is recommended that the certs be replaced at least once every 2 years.\nThe above script generates keys client.key, server.key and certs client.crt, server.crt for both the client and server. The CA cert is generated as root.crt.\nLoad certs into the HTTP/GRPC server/client Every HTTP/GRPC link between Cortex components supports TLS configuration through the following config parameters:\nServer flags # Path to the TLS Cert for the HTTP Server -server.http-tls-cert-path=/path/to/server.crt # Path to the TLS Key for the HTTP Server -server.http-tls-key-path=/path/to/server.key # Type of Client Auth for the HTTP Server -server.http-tls-client-auth=\"RequireAndVerifyClientCert\" # Path to the Client CA Cert for the HTTP Server -server.http-tls-ca-path=\"/path/to/root.crt\" # Path to the TLS Cert for the GRPC Server -server.grpc-tls-cert-path=/path/to/server.crt # Path to the TLS Key for the GRPC Server -server.grpc-tls-key-path=/path/to/server.key # Type of Client Auth for the GRPC Server -server.grpc-tls-client-auth=\"RequireAndVerifyClientCert\" # Path to the Client CA Cert for the GRPC Server -server.grpc-tls-ca-path=/path/to/root.crt Client flags Client flags are component-specific.\nFor an HTTP client in the Alertmanager:\n# Path to the TLS Cert for the HTTP Client -alertmanager.configs.tls-cert-path=/path/to/client.crt # Path to the TLS Key for the HTTP Client -alertmanager.configs.tls-key-path=/path/to/client.key # Path to the TLS CA for the HTTP Client -alertmanager.configs.tls-ca-path=/path/to/root.crt For a GRPC client in the Querier:\n# Path to the TLS Cert for the GRPC Client -querier.frontend-client.tls-cert-path=/path/to/client.crt # Path to the TLS Key for the GRPC Client -querier.frontend-client.tls-key-path=/path/to/client.key # Path to the TLS CA for the GRPC Client -querier.frontend-client.tls-ca-path=/path/to/root.crt Similarly, for the GRPC Ingester Client:\n# Path to the TLS Cert for the GRPC Client -ingester.client.tls-cert-path=/path/to/client.crt # Path to the TLS Key for the GRPC Client -ingester.client.tls-key-path=/path/to/client.key # Path to the TLS CA for the GRPC Client -ingester.client.tls-ca-path=/path/to/root.crt TLS can be configured in a similar fashion for other HTTP/GRPC clients in Cortex.\n","categories":"","description":"","excerpt":"Cortex is a distributed system with significant traffic between its …","ref":"/docs/guides/tls/","tags":"","title":"Securing communication between Cortex components with TLS"},{"body":"Cortex must be deployed with due care over system configuration, using principles such as “least privilege” to limit any exposure due to flaws in the source code.\nYou must configure authorisation and authentication externally to Cortex; see this guide\nInformation about security disclosures and mailing lists is in the main repo\n","categories":"","description":"","excerpt":"Cortex must be deployed with due care over system configuration, using …","ref":"/docs/guides/security/","tags":"","title":"Security"},{"body":"Cortex leverages sharding techniques to horizontally scale both single and multi-tenant clusters beyond the capacity of a single node.\nBackground The default sharding strategy employed by Cortex distributes the workload across the entire pool of instances running a given service (eg. ingesters). For example, on the write path, each tenant’s series are sharded across all ingesters, regardless of how many active series the tenant has or how many different tenants are in the cluster.\nThe default strategy allows for a fair balance on the resources consumed by each instance (ie. CPU and memory) and to maximise these resources across the cluster.\nHowever, in a multi-tenant cluster, this approach also introduces some downsides:\nAn outage affects all tenants A misbehaving tenant (eg. causing out-of-memory) could affect all other tenants. The goal of shuffle sharding is to provide an alternative sharding strategy to reduce the blast radius of an outage and better isolate tenants.\nWhat is shuffle sharding? Shuffle sharding is a technique used to isolate different tenants’ workloads and to give each tenant a single-tenant experience even if they’re running in a shared cluster. This technique has been publicly shared and clearly explained by AWS in their builders’ library and a reference implementation has been shown in the Route53 Infima library.\nThe idea is to assign each tenant a shard composed of a subset of the Cortex service instances, aiming to minimize the overlapping instances between two different tenants. Shuffle sharding brings the following benefits over the default sharding strategy:\nAn outage on some Cortex cluster instances/nodes will only affect a subset of tenants. A misbehaving tenant will affect only its shard instances. Due to the low overlap of instances between different tenants, it’s statistically quite likely that any other tenant will run on different instances or only a subset of instances will match the affected ones. Shuffle sharding requires no more resources than the default sharding strategy, but instances may be less evenly balanced from time to time.\nLow overlapping instances probability For example, given a Cortex cluster running 50 ingesters and assigning each tenant 4 out of 50 ingesters, shuffling instances between each tenant, we get 230K possible combinations.\nRandomly picking two different tenants, we have the:\n71% chance that they will not share any instance 26% chance that they will share only 1 instance 2.7% chance that they will share 2 instances 0.08% chance that they will share 3 instances Only a 0.0004% chance that their instances will fully overlap Cortex shuffle sharding Cortex currently supports shuffle sharding in the following services:\nIngesters Query-frontend / Query-scheduler Store-gateway Ruler Compactor Shuffle sharding is disabled by default and needs to be explicitly enabled in the configuration.\nGuaranteed properties The Cortex shuffle sharding implementation guarantees the following properties:\nStability\nGiven a consistent state of the hash ring, the shuffle sharding algorithm always selects the same instances for a given tenant, even across different machines. Consistency\nAdding or removing 1 instance from the hash ring leads to only 1 instance changed at most, in each tenant’s shard. Shuffling\nProbabilistically and for a large enough cluster, it ensures that every tenant gets a different set of instances, with a reduced number of overlapping instances between two tenants to improve failure isolation. Zone-awareness\nWhen zone-aware replication is enabled, the subset of instances selected for each tenant contains a balanced number of instances for each availability zone. Ingesters shuffle sharding By default, the Cortex distributor spreads the received series across all running ingesters.\nWhen shuffle sharding is enabled for the ingesters, the distributor and ruler on the write path spread each tenant series across -distributor.ingestion-tenant-shard-size number of ingesters, while on the read path, the querier and ruler queries only the subset of ingesters holding the series for a given tenant.\nThe shard size can be overridden on a per-tenant basis in the limits overrides configuration.\nIngesters write path To enable shuffle-sharding for ingesters on the write path, you need to configure the following CLI flags (or their respective YAML config options) to distributor, ingester, and ruler:\n-distributor.sharding-strategy=shuffle-sharding -distributor.ingestion-tenant-shard-size=\u003csize\u003e\n\u003csize\u003e should be set to the number of ingesters each tenant series should be sharded to. If \u003csize\u003e is greater than the number of available ingesters in the Cortex cluster, the tenant series are sharded across all ingesters. Ingesters read path Assuming shuffle-sharding has been enabled for the write path, to enable shuffle-sharding for ingesters on the read path too, you need to configure the following CLI flags (or their respective YAML config options) to querier and ruler:\n-distributor.sharding-strategy=shuffle-sharding -distributor.ingestion-tenant-shard-size=\u003csize\u003e -querier.shuffle-sharding-ingesters-lookback-period=\u003cperiod\u003e\nQueriers and rulers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since ’now - lookback period’. The configured lookback \u003cperiod\u003e should be greater than or equal to -querier.query-store-after and -querier.query-ingesters-within if set, and greater than the estimated minimum time it takes for the oldest samples stored in a block uploaded by ingester to be discovered and available for querying (3h with the default configuration). Rollout strategy If you’re running a Cortex cluster with shuffle-sharding disabled and you want to enable it for ingesters, the following rollout strategy should be used to avoid missing querying any time-series in the ingesters’ memory:\nEnable ingesters shuffle-sharding on the write path Wait at least -querier.shuffle-sharding-ingesters-lookback-period time Enable ingesters shuffle-sharding on the read path Limitation: decreasing the tenant shard size The current shuffle-sharding implementation in Cortex has a limitation which prevents safely decreasing the tenant shard size if the ingesters shuffle-sharding is enabled on the read path.\nThe problem is that if a tenant’s subring decreases in size, there is currently no way for the queriers and rulers to know how big the tenant subring was previously, and hence they will potentially miss an ingester with data for that tenant. In other words, the lookback mechanism to select the ingesters which may have received series since ’now - lookback period’ doesn’t work correctly if the tenant shard size is decreased.\nThis is deemed an infrequent operation that we considered banning, but a workaround still exists:\nDisable shuffle-sharding on the read path Decrease the configured tenant shard size Wait at least -querier.shuffle-sharding-ingesters-lookback-period time Re-enable shuffle-sharding on the read path Query-frontend and Query-scheduler shuffle sharding By default, all Cortex queriers can execute received queries for a given tenant.\nWhen shuffle sharding is enabled by setting -frontend.max-queriers-per-tenant (or its respective YAML config option) to a value higher than 0 and lower than the number of available queriers, only the specified number of queriers will execute queries for a single tenant.\nNote that this distribution happens in the query-frontend, or query-scheduler if used. When using query-scheduler, the -frontend.max-queriers-per-tenant option must be set for the query-scheduler component. When not using query-frontend (with or without scheduler), this option is not available.\nThe maximum number of queriers can be overridden on a per-tenant basis in the limits overrides configuration.\nThe impact of “query of death” In the event a tenant is repeatedly sending a “query of death” which leads the querier to crash or get killed because of out-of-memory, the crashed querier will get disconnected from the query-frontend or query-scheduler and a new querier will be immediately assigned to the tenant’s shard. This practically invalidates the assumption that shuffle-sharding can be used to contain the blast radius in case of a query of death.\nTo mitigate it, Cortex allows you to configure a delay between when a querier disconnects because of a crash and when the crashed querier is actually removed from the tenant’s shard (and another healthy querier is added as a replacement). A delay of 1 minute may be a reasonable trade-off:\nQuery-frontend: -query-frontend.querier-forget-delay=1m Query-scheduler: -query-scheduler.querier-forget-delay=1m Store-gateway shuffle sharding The Cortex store-gateway – used by the blocks storage – by default spreads each tenant’s blocks across all running store-gateways.\nWhen shuffle sharding is enabled via -store-gateway.sharding-strategy=shuffle-sharding (or its respective YAML config option), each tenant’s blocks will be sharded across a subset of -store-gateway.tenant-shard-size store-gateway instances. This configuration needs to be set to store-gateway, querier, and ruler.\nThe shard size can be overridden on a per-tenant basis setting store_gateway_tenant_shard_size in the limits overrides configuration.\nPlease check out the store-gateway documentation for more information about how it works.\nRuler shuffle sharding Cortex ruler can run in three modes:\nNo sharding at all. This is the most basic mode of the ruler. It is activated by using -ruler.enable-sharding=false (default) and works correctly only if a single ruler is running. In this mode, the Ruler loads all rules for all tenants. Default sharding, activated by using -ruler.enable-sharding=true and -ruler.sharding-strategy=default (default). In this mode, rulers register themselves into the ring. Each ruler will then select and evaluate only those rules that it “owns”. Shuffle sharding, activated by using -ruler.enable-sharding=true and -ruler.sharding-strategy=shuffle-sharding. Similarly to default sharding, rulers use the ring to distribute workload, but rule groups for each tenant can only be evaluated on a limited number of rulers (-ruler.tenant-shard-size, can also be set per tenant as ruler_tenant_shard_size in overrides). Note that when using sharding strategy, each rule group is evaluated by a single ruler only; there is no replication.\nCompactor shuffle sharding Cortex compactor can run in three modes:\nNo sharding at all. This is the most basic mode of the compactor. It is activated by using -compactor.sharding-enabled=false (default). In this mode, every compactor will run every compaction. Default sharding, activated by using -compactor.sharding-enabled=true and -compactor.sharding-strategy=default (default). In this mode, compactors register themselves into the ring. One single tenant will belong to only 1 compactor. Shuffle sharding, activated by using -compactor.sharding-enabled=true and -compactor.sharding-strategy=shuffle-sharding. Similarly to default sharding, but compactions for each tenant can be carried out on multiple compactors (-compactor.tenant-shard-size, can also be set per tenant as compactor_tenant_shard_size in overrides). With shuffle sharding selected as the sharding strategy, a subset of the compactors will be used to handle a user based on the shard size.\nThe idea behind using the shuffle sharding strategy for the compactor is to further enable horizontal scalability and build tolerance for compactions that may take longer than the compaction interval.\nFAQ Does shuffle sharding add additional overhead to the KV store? No, shuffle sharding subrings are computed client-side and are not stored in the ring. KV store sizing still depends primarily on the number of replicas (of any component that uses the ring, e.g. ingesters) and tokens per replica.\nHowever, each tenant’s subring is cached in memory on the client-side, which may slightly increase the memory footprint of certain components (mostly the distributor).\n","categories":"","description":"","excerpt":"Cortex leverages sharding techniques to horizontally scale both single …","ref":"/docs/guides/shuffle-sharding/","tags":"","title":"Shuffle Sharding"},{"body":"Cortex uses Jaeger or OpenTelemetry to implement distributed tracing. We have found tracing invaluable for troubleshooting the behavior of Cortex in production.\nJaeger Dependencies In order to send traces, you will need to set up a Jaeger deployment. A deployment includes either the Jaeger all-in-one binary or else a distributed system of agents, collectors, and queriers. If running on Kubernetes, Jaeger Kubernetes is an excellent resource.\nConfiguration In order to configure Cortex to send traces, you must do two things:\nSet the JAEGER_AGENT_HOST environment variable in all components to point to your Jaeger agent. This defaults to localhost. Enable sampling in the appropriate components: The Ingester and Ruler self-initiate traces and should have sampling explicitly enabled. Sampling for the Distributor and Query Frontend can be enabled in Cortex or in an upstream service such as your front door. To enable sampling in Cortex components, you can specify either JAEGER_SAMPLER_MANAGER_HOST_PORT for remote sampling or JAEGER_SAMPLER_TYPE and JAEGER_SAMPLER_PARAM to manually set sampling configuration. See the Jaeger Client Go documentation for the full list of environment variables you can configure.\nNote that you must specify one of JAEGER_AGENT_HOST or JAEGER_SAMPLER_MANAGER_HOST_PORT in each component for Jaeger to be enabled, even if you plan to use the default values.\nOpenTelemetry Dependencies In order to send traces, you will need to set up an OpenTelemetry Collector. The collector will be able to send traces to multiple destinations such as AWS X-Ray, Google Cloud, DataDog and [others(https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter). OpenTelemetry Collector provides a helm chart to set up the environment.\nConfiguration See the document on the tracing section in the Configuration file.\nCurrent State Cortex is maintaining backward compatibility with Jaeger support. Cortex has not fully migrated from OpenTracing to OpenTelemetry and is currently using the OpenTracing bridge.\n","categories":"","description":"","excerpt":"Cortex uses Jaeger or OpenTelemetry to implement distributed tracing. …","ref":"/docs/guides/tracing/","tags":"","title":"Tracing"},{"body":"Cortex supports data replication for different services. By default, data is transparently replicated across the whole pool of service instances, regardless of whether these instances are all running within the same availability zone (or data center, or rack) or in different ones.\nIt is completely possible that all the replicas for the given data are held within the same availability zone, even if the Cortex cluster spans multiple zones. Storing multiple replicas for a given data within the same availability zone poses a risk for data loss if there is an outage affecting various nodes within a zone or a full zone outage.\nFor this reason, Cortex optionally supports zone-aware replication. When zone-aware replication is enabled, replicas for the given data are guaranteed to span across different availability zones. This requires the Cortex cluster to run at least in a number of zones equal to the configured replication factor.\nReads from a zone-aware replication enabled Cortex Cluster can withstand zone failures as long as there are no more than floor(replication factor / 2) zones with failing instances.\nThe Cortex services supporting zone-aware replication are:\nDistributors and Ingesters Store-gateways (blocks storage only) Distributors / Ingesters: time-series replication The Cortex time-series replication is used to hold multiple (typically 3) replicas of each time series in the ingesters.\nTo enable the zone-aware replication for the ingesters you should:\nConfigure the availability zone for each ingester via the -ingester.availability-zone CLI flag (or its respective YAML config option) Rollout ingesters to apply the configured zone Enable time-series zone-aware replication via the -distributor.zone-awareness-enabled CLI flag (or its respective YAML config option). Please be aware that this configuration option should be set to distributors, queriers, and rulers. The -distributor.shard-by-all-labels setting has an impact on read availability. When enabled, a metric is sharded across all ingesters, and the querier needs to fetch series from all ingesters. While, when disabled, a metric is sharded only across \u003creplication factor\u003e ingesters.\nIn the event of a large outage impacting ingesters in more than 1 zone, when -distributor.shard-by-all-labels=true, all queries will fail, while when disabled, some queries may still succeed if the ingesters holding the required metric are not impacted by the outage. To learn more about this flag, please refer to distributor arguments.\nStore-gateways: blocks replication The Cortex store-gateway (used only when Cortex is running with the blocks storage) supports blocks sharding, used to horizontally scale blocks in a large cluster without hitting any vertical scalability limit.\nTo enable the zone-aware replication for the store-gateways, please refer to the store-gateway documentation.\nMinimum number of zones For Cortex to function correctly, there must be at least the same number of availability zones as the replication factor. For example, if the replication factor is configured to 3 (default for time-series replication), the Cortex cluster should be spread at least over 3 availability zones.\nIt is safe to have more zones than the replication factor, but it cannot be less. Having fewer availability zones than the replication factor causes a replica write to be missed, and in some cases, the write fails if the availability zones count is too low.\nImpact on unbalanced zones Cortex requires that each zone runs the same number of instances of a given service for which the zone-aware replication is enabled. This guarantees a fair split of the workload across zones.\nOn the contrary, if zones are unbalanced, the zones with a lower number of instances would have a higher pressure on resources utilization (eg. CPU and memory) compared to zones with a higher number of instances.\nImpact on costs Depending on the underlying infrastructure being used, deploying Cortex across multiple availability zones may cause an increase in running costs as most cloud providers charge for inter-availability zone networking. The most significant change would be for a Cortex cluster currently running in a single zone.\n","categories":"","description":"","excerpt":"Cortex supports data replication for different services. By default, …","ref":"/docs/guides/zone-aware-replication/","tags":"","title":"Zone Aware Replication"},{"body":"master / unreleased [CHANGE] OTLP: Change OTLP handler to be consistent with the Prometheus OTLP handler. #6272 target_info metric is enabled by default and can be disabled via -distributor.otlp.disable-target-info=true flag Convert all attributes to labels is disabled by default and can be enabled via -distributor.otlp.convert-all-attributes=true flag You can specify the attributes converted to labels via -distributor.promote-resource-attributes flag. Supported only if -distributor.otlp.convert-all-attributes=false [CHANGE] Change all max async concurrency default values 50 to 3 #6268 [CHANGE] Change default value of -blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency from 50 to 3 #6265 [CHANGE] Enable Compactor and Alertmanager in target all. #6204 [FEATURE] Ruler: Pagination support for List Rules API. #6299 [FEATURE] Query Frontend/Querier: Add protobuf codec -api.querier-default-codec and the option to choose response compression type -querier.response-compression. #5527 [FEATURE] Ruler: Experimental: Add ruler.frontend-address to allow query to query frontends instead of ingesters. #6151 [FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via -ruler.enable-ha-evaluation flag. #6129 [FEATURE] Store Gateway: Add an in-memory chunk cache. #6245 [FEATURE] Chunk Cache: Support multi level cache and add metrics. #6249 [FEATURE] Distributor: Accept multiple HA Tracker pairs in the same request. #6256 [FEATURE] Ruler: Add support for per-user external labels #6340 [ENHANCEMENT] Store Gateway: Add a hedged request to reduce the tail latency. #6388 [ENHANCEMENT] Ingester: Add metrics to track succeed/failed native histograms. #6370 [ENHANCEMENT] Query Frontend/Querier: Add an experimental flag -querier.enable-promql-experimental-functions to enable experimental promQL functions. #6355 [ENHANCEMENT] OTLP: Add -distributor.otlp-max-recv-msg-size flag to limit OTLP request size in bytes. #6333 [ENHANCEMENT] S3 Bucket Client: Add a list objects version configs to configure list api object version. #6280 [ENHANCEMENT] OpenStack Swift: Add application credential configs for Openstack swift object storage backend. #6255 [ENHANCEMENT] Query Frontend: Add new query stats metrics cortex_query_samples_scanned_total and cortex_query_peak_samples to track scannedSamples and peakSample per user. #6228 [ENHANCEMENT] Ingester: Add option ingester.disable-chunk-trimming to disable chunk trimming. #6300 [ENHANCEMENT] Ingester: Add blocks-storage.tsdb.wal-compression-type to support zstd wal compression type. #6232 [ENHANCEMENT] Query Frontend: Add info field to query response. #6207 [ENHANCEMENT] Query Frontend: Add peakSample in query stats response. #6188 [ENHANCEMENT] Ruler: Add new ruler metric cortex_ruler_rule_groups_in_store that is the total rule groups per tenant in store, which can be used to compare with cortex_prometheus_rule_group_rules to count the number of rule groups that are not loaded by a ruler. #5869 [ENHANCEMENT] Ingester/Ring: New READONLY status on ring to be used by Ingester. New ingester API to change mode of ingester #6163 [ENHANCEMENT] Ruler: Add query statistics metrics when –ruler.query-stats-enabled=true. #6173 [ENHANCEMENT] Ingester: Add new API /ingester/all_user_stats which shows loaded blocks, active timeseries and ingestion rate for a specific ingester. #6178 [ENHANCEMENT] Distributor: Add new cortex_reduced_resolution_histogram_samples_total metric to track the number of histogram samples which resolution was reduced. #6182 [ENHANCEMENT] StoreGateway: Implement metadata API limit in queryable. #6195 [ENHANCEMENT] Ingester: Add matchers to ingester LabelNames() and LabelNamesStream() RPC. #6209 [ENHANCEMENT] KV: Add TLS configs to consul. #6374 [ENHANCEMENT] Ingester/Store Gateway Clients: Introduce an experimental HealthCheck handler to quickly fail requests directed to unhealthy targets. #6225 #6257 [ENHANCEMENT] Upgrade build image and Go version to 1.23.2. #6261 #6262 [ENHANCEMENT] Ingester: Introduce a new experimental feature for caching expanded postings on the ingester. #6296 [ENHANCEMENT] Querier/Ruler: Expose store_gateway_consistency_check_max_attempts for max retries when querying store gateway in consistency check. #6276 [ENHANCEMENT] StoreGateway: Add new cortex_bucket_store_chunk_pool_inuse_bytes metric to track the usage in chunk pool. #6310 [ENHANCEMENT] Distributor: Add new cortex_distributor_inflight_client_requests metric to track number of ingester client inflight requests. #6358 [ENHANCEMENT] Distributor: Expose cortex_label_size_bytes native histogram metric. #6372 [ENHANCEMENT] Add new option -server.grpc_server-num-stream-workers to configure the number of worker goroutines that should be used to process incoming streams. #6386 [ENHANCEMENT] Distributor: Return HTTP 5XX instead of HTTP 4XX when instance limits are hit. #6358 [ENHANCEMENT] Ingester: Make sure unregistered ingester joining the ring after WAL replay #6277 [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224 [BUGFIX] Ruler: Allow rule evaluation to complete during shutdown. #6326 [BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled #6271 [BUGFIX] ingester: Fix regression on usage of cortex_ingester_queried_chunks #6398 1.18.1 2024-10-14 [BUGFIX] Backporting upgrade to go 1.22.7 to patch CVE-2024-34155, CVE-2024-34156, CVE-2024-34158 #6217 #6264 1.18.0 2024-09-03 [CHANGE] Ingester: Remove -querier.query-store-for-labels-enabled flag. Querying long-term store for labels is always enabled. #5984 [CHANGE] Server: Instrument cortex_request_duration_seconds metric with native histogram. If native-histograms feature is enabled in monitoring Prometheus then the metric name needs to be updated in your dashboards. #6056 [CHANGE] Distributor/Ingester: Change cortex_distributor_ingester_appends_total, cortex_distributor_ingester_append_failures_total, cortex_distributor_ingester_queries_total, and cortex_distributor_ingester_query_failures_total metrics to use the ingester ID instead of its IP as the label value. #6078 [CHANGE] OTLP: Set AddMetricSuffixes to true to always enable metric name normalization. #6136 [CHANGE] Querier: Deprecate and enable by default querier.ingester-metadata-streaming flag. #6147 [CHANGE] QueryFrontend/QueryScheduler: Deprecate -querier.max-outstanding-requests-per-tenant and -query-scheduler.max-outstanding-requests-per-tenant flags. Use frontend.max-outstanding-requests-per-tenant instead. #6146 [CHANGE] Ingesters: Enable ‘snappy-block’ compression on ingester clients by default. #6148 [CHANGE] Ruler: Scheduling ruler.evaluation-delay-duration to be deprecated. Ruler will use the highest value between ruler.evaluation-delay-duration and ruler.query-offset #6149 [CHANGE] Querier: Remove -querier.at-modifier-enabled flag. #6157 [CHANGE] Tracing: Remove deprecated oltp_endpoint config entirely. #6158 [CHANGE] Store Gateway: Enable store gateway zone stable shuffle sharding by default. #6161 [FEATURE] Ingester/Distributor: Experimental: Enable native histogram ingestion via -blocks-storage.tsdb.enable-native-histograms flag. #5986 #6010 #6020 [FEATURE] Querier: Enable querying native histogram chunks. #5944 #6031 [FEATURE] Query Frontend: Support native histogram in query frontend response. #5996 #6043 [FEATURE] Ruler: Support sending native histogram samples to Ingester. #6029 [FEATURE] Ruler: Add support for filtering out alerts in ListRules API. #6011 [FEATURE] Query Frontend: Added a query rejection mechanism to block resource-intensive queries. #6005 [FEATURE] OTLP: Support ingesting OTLP exponential metrics as native histograms. #6071 #6135 [FEATURE] Ingester: Add ingester.instance-limits.max-inflight-query-requests to allow limiting ingester concurrent queries. #6081 [FEATURE] Distributor: Add validation.max-native-histogram-buckets to limit max number of bucket count. Distributor will try to automatically reduce histogram resolution until it is within the bucket limit or resolution cannot be reduced anymore. #6104 [FEATURE] Store Gateway: Introduce token bucket limiter to enhance store gateway throttling. #6016 [FEATURE] Ruler: Add support for query_offset field on RuleGroup and new ruler_query_offset per-tenant limit. #6085 [ENHANCEMENT] Ruler: Add support to persist tokens in rulers. #5987 [ENHANCEMENT] Query Frontend/Querier: Added store gateway postings touched count and touched size in Querier stats and log in Query Frontend. #5892 [ENHANCEMENT] Query Frontend/Querier: Returns warnings on prometheus query responses. #5916 [ENHANCEMENT] Ingester: Allowing to configure -blocks-storage.tsdb.head-compaction-interval flag up to 30 min and add a jitter on the first head compaction. #5919 #5928 [ENHANCEMENT] Distributor: Added max_inflight_push_requests config to ingester client to protect distributor from OOMKilled. #5917 [ENHANCEMENT] Distributor/Querier: Clean stale per-ingester metrics after ingester restarts. #5930 [ENHANCEMENT] Distributor/Ring: Allow disabling detailed ring metrics by ring member. #5931 [ENHANCEMENT] KV: Etcd Added etcd.ping-without-stream-allowed parameter to disable/enable PermitWithoutStream #5933 [ENHANCEMENT] Ingester: Add a new limits_per_label_set limit. This limit functions similarly to max_series_per_metric, but allowing users to define the maximum number of series per LabelSet. #5950 #5993 [ENHANCEMENT] Store Gateway: Log gRPC requests together with headers configured in http_request_headers_to_log. #5958 [ENHANCEMENT] Ingester: Add a new experimental -ingester.labels-string-interning-enabled flag to enable string interning for metrics labels. #6057 [ENHANCEMENT] Ingester: Add link to renew 10% of the ingesters tokens in the admin page. #6063 [ENHANCEMENT] Ruler: Add support for filtering by state and health field on Rules API. #6040 [ENHANCEMENT] Ruler: Add support for filtering by match field on Rules API. #6083 [ENHANCEMENT] Distributor: Reduce memory usage when error volume is high. #6095 [ENHANCEMENT] Compactor: Centralize metrics used by compactor and add user label to compactor metrics. #6096 [ENHANCEMENT] Compactor: Add unique execution ID for each compaction cycle in log for easy debugging. #6097 [ENHANCEMENT] Compactor: Differentiate retry and halt error and retry failed compaction only on retriable error. #6111 [ENHANCEMENT] Ruler: Add support for filtering by state and health field on Rules API. #6040 [ENHANCEMENT] Compactor: Split cleaner cycle for active and deleted tenants. #6112 [ENHANCEMENT] Compactor: Introduce cleaner visit marker. #6113 [ENHANCEMENT] Query Frontend: Add cortex_query_samples_total metric. #6142 [ENHANCEMENT] Ingester: Implement metadata API limit. #6128 [BUGFIX] Configsdb: Fix endline issue in db password. #5920 [BUGFIX] Ingester: Fix user and type labels for the cortex_ingester_tsdb_head_samples_appended_total TSDB metric. #5952 [BUGFIX] Querier: Enforce max query length check for /api/v1/series API even though ignoreMaxQueryLength is set to true. #6018 [BUGFIX] Ingester: Fix issue with the minimize token generator where it was not taking in consideration the current ownership of an instance when generating extra tokens. #6062 [BUGFIX] Scheduler: Fix user queue in scheduler that was not thread-safe. #6077 #6160 [BUGFIX] Ingester: Include out-of-order head compaction when compacting TSDB head. #6108 [BUGFIX] Ingester: Fix cortex_ingester_tsdb_mmap_chunks_total metric. #6134 [BUGFIX] Query Frontend: Fix query rejection bug for metadata queries. #6143 1.17.1 2024-05-20 [CHANGE] Query Frontend/Ruler: Omit empty data, errorType and error fields in API response. #5953 #5954 [ENHANCEMENT] Ingester: Added upload_compacted_blocks_enabled config to ingester to parameterize uploading compacted blocks. #5959 [BUGFIX] Querier: Select correct tenant during query federation. #5943 1.17.0 2024-04-30 [CHANGE] Azure Storage: Upgraded objstore dependency and support Azure Workload Identity Authentication. Added connection_string to support authenticating via SAS token. Marked msi_resource config as deprecating. #5645 [CHANGE] Store Gateway: Add a new fastcache based inmemory index cache. #5619 [CHANGE] Index Cache: Multi level cache backfilling operation becomes async. Added -blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency and -blocks-storage.bucket-store.index-cache.multilevel.max-async-buffer-size configs and metric cortex_store_multilevel_index_cache_backfill_dropped_items_total for number of dropped items. #5661 [CHANGE] Ingester: Disable uploading compacted blocks and overlapping compaction in ingester. #5735 [CHANGE] Distributor: Count the number of rate-limited samples in distributor_samples_in_total. #5714 [CHANGE] Ruler: Remove cortex_ruler_write_requests_total, cortex_ruler_write_requests_failed_total, cortex_ruler_queries_total, cortex_ruler_queries_failed_total, and cortex_ruler_query_seconds_total metrics for the tenant when the ruler deletes the manager for the tenant. #5772 [CHANGE] Main: Mark mem-ballast-size-bytes flag as deprecated. #5816 [CHANGE] Querier: Mark -querier.ingester-streaming flag as deprecated. Now query ingester streaming is always enabled. #5817 [CHANGE] Compactor/Bucket Store: Added -blocks-storage.bucket-store.block-discovery-strategy to configure different block listing strategy. Reverted the current recursive block listing mechanism and use the strategy Concurrent as in 1.15. #5828 [CHANGE] Compactor: Don’t halt compactor when overlapped source blocks detected. #5854 [CHANGE] S3 Bucket Client: Expose -blocks-storage.s3.send-content-md5 flag and set default checksum algorithm to MD5. #5870 [CHANGE] Querier: Mark querier.iterators and querier.batch-iterators flags as deprecated. Now querier always use batch iterators. #5868 [CHANGE] Query Frontend: Error response returned by Query Frontend now follows Prometheus API error response format. #5811 [FEATURE] Experimental: OTLP ingestion. #5813 [FEATURE] Query Frontend/Scheduler: Add query priority support. #5605 [FEATURE] Tracing: Use kuberesolver to resolve OTLP endpoints address with kubernetes:// prefix as Kubernetes service. #5731 [FEATURE] Tracing: Add tracing.otel.round-robin flag to use round_robin gRPC client side LB policy for sending OTLP traces. #5731 [FEATURE] Ruler: Add ruler.concurrent-evals-enabled flag to enable concurrent evaluation within a single rule group for independent rules. Maximum concurrency can be configured via ruler.max-concurrent-evals. #5766 [FEATURE] Distributor Queryable: Experimental: Add config zone_results_quorum_metadata. When querying ingesters using metadata APIs such as label names and values, only results from quorum number of zones will be included and merged. #5779 [FEATURE] Storage Cache Clients: Add config set_async_circuit_breaker_config to utilize the circuit breaker pattern for dynamically thresholding asynchronous set operations. Implemented in both memcached and redis cache clients. #5789 [FEATURE] Ruler: Add experimental experimental.ruler.api-deduplicate-rules flag to remove duplicate rule groups from the Prometheus compatible rules API endpoint. Add experimental ruler.ring.replication-factor and ruler.ring.zone-awareness-enabled flags to configure rule group replication, but only the first ruler in the replicaset evaluates the rule group, the rest will be used as backup to handle events when a ruler is down during an API request to list rules. #5782 #5901 [FEATURE] Ring: Add experimental -ingester.tokens-generator-strategy=minimize-spread flag to enable the new minimize spread token generator strategy. #5855 [FEATURE] Ring Status Page: Add Ownership Diff From Expected column in the ring table to indicate the extent to which the ownership of a specific ingester differs from the expected ownership. #5889 [ENHANCEMENT] Ingester: Add per-tenant new metric cortex_ingester_tsdb_data_replay_duration_seconds. #5477 [ENHANCEMENT] Store Gateway: Added -store-gateway.enabled-tenants and -store-gateway.disabled-tenants to explicitly enable or disable store-gateway for specific tenants. #5638 [ENHANCEMENT] Query Frontend: Write service timing header in response even though there is an error. #5653 [ENHANCEMENT] Compactor: Add new compactor metric cortex_compactor_start_duration_seconds. #5683 [ENHANCEMENT] Index Cache: Multi level cache adds config max_backfill_items to cap max items to backfill per async operation. #5686 [ENHANCEMENT] Query Frontend: Log number of split queries in query stats log. #5703 [ENHANCEMENT] Compactor: Skip compaction retry when encountering a permission denied error. #5727 [ENHANCEMENT] Logging: Added new options for logging HTTP request headers: -server.log-request-headers enables logging HTTP request headers, -server.log-request-headers-exclude-list allows users to specify headers which should not be logged. #5744 [ENHANCEMENT] Query Frontend/Scheduler: Time check in query priority now considers overall data select time window (including range selectors, modifiers and lookback delta). #5758 [ENHANCEMENT] Querier: Added querier.store-gateway-query-stats-enabled to enable or disable store gateway query stats log. #5749 [ENHANCEMENT] Querier: Improve labels APIs latency by merging slices using K-way merge and more than 1 core. #5785 [ENHANCEMENT] AlertManager: Retrying AlertManager Delete Silence on error. #5794 [ENHANCEMENT] Ingester: Add new ingester metric cortex_ingester_max_inflight_query_requests. #5798 [ENHANCEMENT] Query: Added query_storage_wall_time to Query Frontend and Ruler query stats log for wall time spent on fetching data from storage. Query evaluation is not included. #5799 [ENHANCEMENT] Query: Added additional max query length check at Query Frontend and Ruler. Added -querier.ignore-max-query-length flag to disable max query length check at Querier. #5808 [ENHANCEMENT] Querier: Add context error check when converting Metrics to SeriesSet for GetSeries on distributorQuerier. #5827 [ENHANCEMENT] Ruler: Improve GetRules response time by reducing lock contention and introducing a temporary rules cache in ruler/manager.go. #5805 [ENHANCEMENT] Querier: Add context error check when merging slices from ingesters for GetLabel operations. #5837 [BUGFIX] Distributor: Do not use label with empty values for sharding #5717 [BUGFIX] Query Frontend: queries with negative offset should check whether it is cacheable or not. #5719 [BUGFIX] Redis Cache: pass cache_size config correctly. #5734 [BUGFIX] Distributor: Shuffle-Sharding with ingestion_tenant_shard_size set to 0, default sharding strategy should be used. #5189 [BUGFIX] Cortex: Fix GRPC stream clients not honoring overrides for call options. #5797 [BUGFIX] Ruler: Fix support for keep_firing_for field in alert rules. #5823 [BUGFIX] Ring DDB: Fix lifecycle for ring counting unhealthy pods as healthy. #5838 [BUGFIX] Ring DDB: Fix region assignment. #5842 1.16.1 2024-04-23 [ENHANCEMENT] Upgraded Docker base images to alpine:3.18. #5684 [ENHANCEMENT] Upgrade to go 1.21.9 #5879 #5882 1.16.0 2023-11-20 [CHANGE] AlertManager: include reason label in cortex_alertmanager_notifications_failed_total. #5409 [CHANGE] Ruler: Added user label to cortex_ruler_write_requests_total, cortex_ruler_write_requests_failed_total, cortex_ruler_queries_total, and cortex_ruler_queries_failed_total metrics. #5312 [CHANGE] Alertmanager: Validating new fields on the PagerDuty AM config. #5290 [CHANGE] Ingester: Creating label native-histogram-sample on the cortex_discarded_samples_total to keep track of discarded native histogram samples. #5289 [CHANGE] Store Gateway: Rename cortex_bucket_store_cached_postings_compression_time_seconds to cortex_bucket_store_cached_postings_compression_time_seconds_total. #5431 [CHANGE] Store Gateway: Rename cortex_bucket_store_cached_series_fetch_duration_seconds to cortex_bucket_store_series_fetch_duration_seconds and cortex_bucket_store_cached_postings_fetch_duration_seconds to cortex_bucket_store_postings_fetch_duration_seconds. Add new metric cortex_bucket_store_chunks_fetch_duration_seconds. #5448 [CHANGE] Store Gateway: Remove idle_timeout, max_conn_age, pool_size, min_idle_conns fields for Redis index cache and caching bucket. #5448 [CHANGE] Store Gateway: Add flag -store-gateway.sharding-ring.zone-stable-shuffle-sharding to enable store gateway to use zone stable shuffle sharding. #5489 [CHANGE] Bucket Index: Add series_max_size and chunk_max_size to bucket index. #5489 [CHANGE] StoreGateway: Rename cortex_bucket_store_chunk_pool_returned_bytes_total and cortex_bucket_store_chunk_pool_requested_bytes_total to cortex_bucket_store_chunk_pool_operation_bytes_total. #5552 [CHANGE] Query Frontend/Querier: Make build info API disabled by default and add feature flag api.build-info-enabled to enable it. #5533 [CHANGE] Purger: Do no use S3 tenant kms key when uploading deletion marker. #5575 [CHANGE] Ingester: Shipper always allows uploading compacted blocks to ship OOO compacted blocks. #5625 [CHANGE] DDBKV: Change metric name from dynamodb_kv_read_capacity_total to dynamodb_kv_consumed_capacity_total and include Delete, Put, Batch dimension. #5487 [CHANGE] Compactor: Adding the userId on the compact dir path. #5524 [CHANGE] Ingester: Remove deprecated ingester metrics. #5472 [CHANGE] Query Frontend: Expose -querier.max-subquery-steps to configure subquery max steps check. By default, the limit is set to 0, which is disabled. #5656 [FEATURE] Store Gateway: Implementing multi level index cache. #5451 [FEATURE] Ruler: Add support for disabling rule groups. #5521 [FEATURE] Support object storage backends for runtime configuration file. #5292 [FEATURE] Ruler: Add support for Limit field on RuleGroup. #5528 [FEATURE] AlertManager: Add support for Webex, Discord and Telegram Receiver. #5493 [FEATURE] Ingester: added -admin-limit-message to customize the message contained in limit errors.#5460 [FEATURE] AlertManager: Update version to v0.26.0 and bring in Microsoft Teams receiver. #5543 [FEATURE] Store Gateway: Support lazy expanded posting optimization. Added new flag blocks-storage.bucket-store.lazy-expanded-postings-enabled and new metrics cortex_bucket_store_lazy_expanded_postings_total, cortex_bucket_store_lazy_expanded_posting_size_bytes_total and cortex_bucket_store_lazy_expanded_posting_series_overfetched_size_bytes_total. #5556. [FEATURE] Store Gateway: Add max_downloaded_bytes_per_request to limit max bytes to download per store gateway request. #5179 [FEATURE] Added 2 flags -alertmanager.alertmanager-client.grpc-max-send-msg-size and -alertmanager.alertmanager-client.grpc-max-recv-msg-size to configure alert manager grpc client message size limits. #5338 [FEATURE] Querier/StoreGateway: Allow the tenant shard sizes to be a percent of total instances. #5393 [FEATURE] Added the flag -alertmanager.api-concurrency to configure alert manager api concurrency limit. #5412 [FEATURE] Store Gateway: Add -store-gateway.sharding-ring.keep-instance-in-the-ring-on-shutdown to skip unregistering instance from the ring in shutdown. #5421 [FEATURE] Ruler: Support for filtering rules in the API. #5417 [FEATURE] Compactor: Add -compactor.ring.tokens-file-path to store generated tokens locally. #5432 [FEATURE] Query Frontend: Add -frontend.retry-on-too-many-outstanding-requests to re-enqueue 429 requests if there are multiple query-schedulers available. #5496 [FEATURE] Store Gateway: Add -blocks-storage.bucket-store.max-inflight-requests for store gateways to reject further series requests upon reaching the limit. #5553 [FEATURE] Store Gateway: Support filtered index cache. #5587 [ENHANCEMENT] Update go version to 1.21.3. #5630 [ENHANCEMENT] Store Gateway: Add cortex_bucket_store_block_load_duration_seconds histogram to track time to load blocks. #5580 [ENHANCEMENT] Querier: retry chunk pool exhaustion error in querier rather than query frontend. #5569 [ENHANCEMENT] Alertmanager: Added flag -alertmanager.alerts-gc-interval to configure alerts Garbage collection interval. #5550 [ENHANCEMENT] Query Frontend: enable vertical sharding on binary expr . #5507 [ENHANCEMENT] Query Frontend: Include user agent as part of query frontend log. #5450 [ENHANCEMENT] Query: Set CORS Origin headers for Query API #5388 [ENHANCEMENT] Query Frontend: Add cortex_rejected_queries_total metric for throttled queries. #5356 [ENHANCEMENT] Query Frontend: Optimize the decoding of SampleStream. #5349 [ENHANCEMENT] Compactor: Check ctx done when uploading visit marker. #5333 [ENHANCEMENT] AlertManager: Add cortex_alertmanager_dispatcher_aggregation_groups and cortex_alertmanager_dispatcher_alert_processing_duration_seconds metrics for dispatcher. #5592 [ENHANCEMENT] Store Gateway: Added new flag blocks-storage.bucket-store.series-batch-size to control how many series to fetch per batch in Store Gateway. #5582. [ENHANCEMENT] Querier: Log query stats when querying store gateway. #5376 [ENHANCEMENT] Ruler: Add cortex_ruler_rule_group_load_duration_seconds and cortex_ruler_rule_group_sync_duration_seconds metrics. #5609 [ENHANCEMENT] Ruler: Add contextual info and query statistics to log #5604 [ENHANCEMENT] Distributor/Ingester: Add span on push path #5319 [ENHANCEMENT] Query Frontend: Reject subquery with too small step size. #5323 [ENHANCEMENT] Compactor: Exposing Thanos accept-malformed-index to Cortex compactor. #5334 [ENHANCEMENT] Log: Avoid expensive log.Valuer evaluation for disallowed levels. #5297 [ENHANCEMENT] Improving Performance on the API Gzip Handler. #5347 [ENHANCEMENT] Dynamodb: Add puller-sync-time to allow different pull time for ring. #5357 [ENHANCEMENT] Emit querier max_concurrent as a metric. #5362 [ENHANCEMENT] Avoid sort tokens on lifecycler autoJoin. #5394 [ENHANCEMENT] Do not resync blocks in running store gateways during rollout deployment and container restart. #5363 [ENHANCEMENT] Store Gateway: Add new metrics cortex_bucket_store_sent_chunk_size_bytes, cortex_bucket_store_postings_size_bytes and cortex_bucket_store_empty_postings_total. #5397 [ENHANCEMENT] Add jitter to lifecycler heartbeat. #5404 [ENHANCEMENT] Store Gateway: Add config estimated_max_series_size_bytes and estimated_max_chunk_size_bytes to address data overfetch. #5401 [ENHANCEMENT] Distributor/Ingester: Add experimental -distributor.sign_write_requests flag to sign the write requests. #5430 [ENHANCEMENT] Store Gateway/Querier/Compactor: Handling CMK Access Denied errors. #5420 #5442 #5446 [ENHANCEMENT] Alertmanager: Add the alert name in error log when it get throttled. #5456 [ENHANCEMENT] Querier: Retry store gateway on different zones when zone awareness is enabled. #5476 [ENHANCEMENT] Compactor: allow unregister_on_shutdown to be configurable. #5503 [ENHANCEMENT] Querier: Batch adding series to query limiter to optimize locking. #5505 [ENHANCEMENT] Store Gateway: add metric cortex_bucket_store_chunk_refetches_total for number of chunk refetches. #5532 [ENHANCEMENT] BasicLifeCycler: allow final-sleep during shutdown #5517 [ENHANCEMENT] All: Handling CMK Access Denied errors. #5420 #5542 [ENHANCEMENT] Querier: Retry store gateway client connection closing gRPC error. #5558 [ENHANCEMENT] QueryFrontend: Add generic retry for all APIs. #5561. [ENHANCEMENT] Querier: Check context before notifying scheduler and frontend. #5565 [ENHANCEMENT] QueryFrontend: Add metric for number of series requests. #5373 [ENHANCEMENT] Store Gateway: Add histogram metrics for total time spent fetching series and chunks per request. #5573 [ENHANCEMENT] Store Gateway: Check context in multi level cache. Add cortex_store_multilevel_index_cache_fetch_duration_seconds and cortex_store_multilevel_index_cache_backfill_duration_seconds to measure fetch and backfill latency. #5596 [ENHANCEMENT] Ingester: Added new ingester TSDB metrics cortex_ingester_tsdb_head_samples_appended_total, cortex_ingester_tsdb_head_out_of_order_samples_appended_total, cortex_ingester_tsdb_snapshot_replay_error_total, cortex_ingester_tsdb_sample_ooo_delta and cortex_ingester_tsdb_mmap_chunks_total. #5624 [ENHANCEMENT] Query Frontend: Handle context error before decoding and merging responses. #5499 [ENHANCEMENT] Store-Gateway and AlertManager: Add a wait_instance_time_out to context to avoid waiting forever. #5581 [ENHANCEMENT] Blocks storage: Move the tenant deletion mark from \u003ctenantID\u003e/markers/tenant-deletion-mark.json to __markers__/\u003ctenantID\u003e/tenant-deletion-mark.json. #5676 [BUGFIX] Compactor: Fix possible division by zero during compactor config validation. #5535 [BUGFIX] Ruler: Validate if rule group can be safely converted back to rule group yaml from protobuf message #5265 [BUGFIX] Querier: Convert gRPC ResourceExhausted status code from store gateway to 422 limit error. #5286 [BUGFIX] Alertmanager: Route web-ui requests to the alertmanager distributor when sharding is enabled. #5293 [BUGFIX] Storage: Bucket index updater should ignore meta not found for partial blocks. #5343 [BUGFIX] Ring: Add JOINING state to read operation. #5346 [BUGFIX] Compactor: Partial block with only visit marker should be deleted even there is no deletion marker. #5342 [BUGFIX] KV: Etcd calls will no longer block indefinitely and will now time out after the DialTimeout period. #5392 [BUGFIX] Ring: Allow RF greater than number of zones to select more than one instance per zone #5411 [BUGFIX] Store Gateway: Fix bug in store gateway ring comparison logic. #5426 [BUGFIX] Ring: Fix bug in consistency of Get func in a scaling zone-aware ring. #5429 [BUGFIX] Compactor: Fix retry on markers. #5441 [BUGFIX] Query Frontend: Fix bug of failing to cancel downstream request context in query frontend v2 mode (query scheduler enabled). #5447 [BUGFIX] Alertmanager: Remove the user id from state replication key metric label value. #5453 [BUGFIX] Compactor: Avoid cleaner concurrency issues checking global markers before all blocks. #5457 [BUGFIX] DDBKV: Disallow instance with older timestamp to update instance with newer timestamp. #5480 [BUGFIX] DDBKV: When no change detected in ring, retry the CAS until there is change. #5502 [BUGFIX] Fix bug on objstore when configured to use S3 fips endpoints. #5540 [BUGFIX] Ruler: Fix bug on ruler where a failure to load a single RuleGroup would prevent rulers to sync all RuleGroup. #5563 [BUGFIX] Query Frontend: Fix query string being omitted in query stats log. #5655 1.15.3 2023-06-22 [BUGFIX] Distributor: Fix potential data corruption in cases of timeout between distributors and ingesters. #5422 1.15.2 2023-05-09 [ENHANCEMENT] Update Go version to 1.20.4. #5299 1.15.1 2023-04-26 [CHANGE] Alertmanager: Validating new fields on the PagerDuty AM config. #5290 [BUGFIX] Querier: Convert gRPC ResourceExhausted status code from store gateway to 422 limit error. #5286 1.15.0 2023-04-19 [CHANGE] Storage: Make Max exemplars config per tenant instead of global configuration. #5080 #5122 [CHANGE] Alertmanager: Local file disclosure vulnerability in OpsGenie configuration has been fixed. #5045 [CHANGE] Rename oltp_endpoint to otlp_endpoint to match opentelemetry spec and lib name. #5068 [CHANGE] Distributor/Ingester: Log warn level on push requests when they have status code 4xx. Do not log if status is 429. #5103 [CHANGE] Tracing: Use the default OTEL trace sampler when -tracing.otel.exporter-type is set to awsxray. #5141 [CHANGE] Ingester partial error log line to debug level. #5192 [CHANGE] Change HTTP status code from 503/422 to 499 if a request is canceled. #5220 [CHANGE] Store gateways summary metrics have been converted to histograms cortex_bucket_store_series_blocks_queried, cortex_bucket_store_series_data_fetched, cortex_bucket_store_series_data_size_touched_bytes, cortex_bucket_store_series_data_size_fetched_bytes, cortex_bucket_store_series_data_touched, cortex_bucket_store_series_result_series #5239 [FEATURE] Querier/Query Frontend: support Prometheus /api/v1/status/buildinfo API. #4978 [FEATURE] Ingester: Add active series to all_user_stats page. #4972 [FEATURE] Ingester: Added -blocks-storage.tsdb.head-chunks-write-queue-size allowing to configure the size of the in-memory queue used before flushing chunks to the disk . #5000 [FEATURE] Query Frontend: Log query params in query frontend even if error happens. #5005 [FEATURE] Ingester: Enable snapshotting of In-memory TSDB on disk during shutdown via -blocks-storage.tsdb.memory-snapshot-on-shutdown. #5011 [FEATURE] Query Frontend/Scheduler: Add a new counter metric cortex_request_queue_requests_total for total requests going to queue. #5030 [FEATURE] Build ARM docker images. #5041 [FEATURE] Query-frontend/Querier: Create spans to measure time to merge promql responses. #5041 [FEATURE] Querier/Ruler: Support the new thanos promql engine. This is an experimental feature and might change in the future. #5093 [FEATURE] Added zstd as an option for grpc compression #5092 [FEATURE] Ring: Add new kv store option dynamodb. #5026 [FEATURE] Cache: Support redis as backend for caching bucket and index cache. #5057 [FEATURE] Querier/Store-Gateway: Added -blocks-storage.bucket-store.ignore-blocks-within allowing to filter out the recently created blocks from being synced by queriers and store-gateways. #5166 [FEATURE] AlertManager/Ruler: Added support for keep_firing_for on alerting rulers. [FEATURE] Alertmanager: Add support for time_intervals. #5102 [FEATURE] Added snappy-block as an option for grpc compression #5215 [FEATURE] Enable experimental out-of-order samples support. Added 2 new configs ingester.out_of_order_time_window and blocks-storage.tsdb.out_of_order_cap_max. #4964 [ENHANCEMENT] Querier: limit series query to only ingesters if start param is not specified. #4976 [ENHANCEMENT] Query-frontend/scheduler: add a new limit frontend.max-outstanding-requests-per-tenant for configuring queue size per tenant. Started deprecating two flags -query-scheduler.max-outstanding-requests-per-tenant and -querier.max-outstanding-requests-per-tenant, and change their value default to 0. Now if both the old flag and new flag are specified, the old flag’s queue size will be picked. #4991 [ENHANCEMENT] Query-tee: Add /api/v1/query_exemplars API endpoint support. #5010 [ENHANCEMENT] Let blocks_cleaner delete blocks concurrently(default 16 goroutines). #5028 [ENHANCEMENT] Query Frontend/Query Scheduler: Increase upper bound to 60s for queue duration histogram metric. #5029 [ENHANCEMENT] Query Frontend: Log Vertical sharding information when query_stats_enabled is enabled. #5037 [ENHANCEMENT] Ingester: The metadata APIs should honour querier.query-ingesters-within when querier.query-store-for-labels-enabled is true. #5027 [ENHANCEMENT] Query Frontend: Skip instant query roundtripper if sharding is not applicable. #5062 [ENHANCEMENT] Push reduce one hash operation of Labels. #4945 #5114 [ENHANCEMENT] Alertmanager: Added -alertmanager.enabled-tenants and -alertmanager.disabled-tenants to explicitly enable or disable alertmanager for specific tenants. #5116 [ENHANCEMENT] Upgraded Docker base images to alpine:3.17. #5132 [ENHANCEMENT] Add retry logic to S3 bucket client. #5135 [ENHANCEMENT] Update Go version to 1.20.1. #5159 [ENHANCEMENT] Distributor: Reuse byte slices when serializing requests from distributors to ingesters. #5193 [ENHANCEMENT] Query Frontend: Add number of chunks and samples fetched in query stats. #5198 [ENHANCEMENT] Implement grpc.Compressor.DecompressedSize for snappy to optimize memory allocations. #5213 [ENHANCEMENT] Querier: Batch Iterator optimization to prevent transversing it multiple times query ranges steps does not overlap. #5237 [BUGFIX] Updated golang.org/x/net dependency to fix CVE-2022-27664. #5008 [BUGFIX] Fix panic when otel and xray tracing is enabled. #5044 [BUGFIX] Fixed no compact block got grouped in shuffle sharding grouper. #5055 [BUGFIX] Fixed ingesters with less tokens stuck in LEAVING. #5061 [BUGFIX] Tracing: Fix missing object storage span instrumentation. #5074 [BUGFIX] Ingester: Fix Ingesters returning empty response for metadata APIs. #5081 [BUGFIX] Ingester: Fix panic when querying metadata from blocks that are being deleted. #5119 [BUGFIX] Ring: Fix case when dynamodb kv reaches the limit of 25 actions per batch call. #5136 [BUGFIX] Query-frontend: Fix shardable instant queries do not produce sorted results for sort, sort_desc, topk, bottomk functions. #5148, #5170 [BUGFIX] Querier: Fix /api/v1/series returning 5XX instead of 4XX when limits are hit. #5169 [BUGFIX] Compactor: Fix issue that shuffle sharding planner return error if block is under visit by other compactor. #5188 [BUGFIX] Fix S3 BucketWithRetries upload empty content issue #5217 [BUGFIX] Query Frontend: Disable absent, absent_over_time and scalar for vertical sharding. #5221 [BUGFIX] Catch context error in the s3 bucket client. #5240 [BUGFIX] Fix query frontend remote read empty body. #5257 [BUGFIX] Fix query frontend incorrect error response format at SplitByQuery middleware. #5260 1.14.0 2022-12-02 This release removes support for chunks storage. See below for more.\n[CHANGE] Remove support for chunks storage entirely. If you are using chunks storage on a previous version, you must migrate your data on version 1.13.1 or earlier. Before upgrading to this release, you should also remove any deprecated chunks-related configuration, as this release will no longer accept that. The following flags are gone: -dynamodb.* -metrics.* -s3.* -azure.* -bigtable.* -gcs.* -cassandra.* -boltdb.* -local.* some -ingester flags: -ingester.wal-enabled -ingester.checkpoint-enabled -ingester.recover-from-wal -ingester.wal-dir -ingester.checkpoint-duration -ingester.flush-on-shutdown-with-wal-enabled -ingester.max-transfer-retries -ingester.max-samples-per-query -ingester.min-chunk-length -ingester.flush-period -ingester.retain-period -ingester.max-chunk-idle -ingester.max-stale-chunk-idle -ingester.flush-op-timeout -ingester.max-chunk-age -ingester.chunk-age-jitter -ingester.concurrent-flushes -ingester.spread-flushes -ingester.chunk-encoding -store.* except -store.engine and -store.max-query-length -store.query-chunk-limit was deprecated and replaced by -querier.max-fetched-chunks-per-query -deletes.* -grpc-store.* -flusher.wal-dir, -flusher.concurrent-flushes, -flusher.flush-op-timeout [CHANGE] Remove support for alertmanager and ruler legacy store configuration. Before upgrading, you need to convert your configuration to use the alertmanager-storage and ruler-storage configuration on the version that you’re already running, then upgrade. [CHANGE] Disables TSDB isolation. #4825 [CHANGE] Drops support Prometheus 1.x rule format on configdb. #4826 [CHANGE] Removes -ingester.stream-chunks-when-using-blocks experimental flag and stream chunks by default when querier.ingester-streaming is enabled. #4864 [CHANGE] Compactor: Added cortex_compactor_runs_interrupted_total to separate compaction interruptions from failures [CHANGE] Enable PromQL @ modifier, negative offset always. #4927 [CHANGE] Store-gateway: Add user label to cortex_bucket_store_blocks_loaded metric. #4918 [CHANGE] AlertManager: include status label in cortex_alertmanager_alerts_received_total. #4907 [FEATURE] Compactor: Added -compactor.block-files-concurrency allowing to configure number of go routines for download/upload block files during compaction. #4784 [FEATURE] Compactor: Added -compactor.blocks-fetch-concurrency allowing to configure number of go routines for blocks during compaction. #4787 [FEATURE] Compactor: Added configurations for Azure MSI in blocks-storage, ruler-storage and alertmanager-storage. #4818 [FEATURE] Ruler: Add support to pass custom implementations of queryable and pusher. #4782 [FEATURE] Create OpenTelemetry Bridge for Tracing. Now cortex can send traces to multiple destinations using OTEL Collectors. #4834 [FEATURE] Added -api.http-request-headers-to-log allowing for the addition of HTTP Headers to logs #4803 [FEATURE] Distributor: Added a new limit -validation.max-labels-size-bytes allowing to limit the combined size of labels for each timeseries. #4848 [FEATURE] Storage/Bucket: Added -*.s3.bucket-lookup-type allowing to configure the s3 bucket lookup type. #4794 [FEATURE] QueryFrontend: Implement experimental vertical sharding at query frontend for range/instant queries. #4863 [FEATURE] QueryFrontend: Support vertical sharding for subqueries. #4955 [FEATURE] Querier: Added a new limit -querier.max-fetched-data-bytes-per-query allowing to limit the maximum size of all data in bytes that a query can fetch from each ingester and storage. #4854 [FEATURE] Added 2 flags -alertmanager.alertmanager-client.grpc-compression and -querier.store-gateway-client.grpc-compression to configure compression methods for grpc clients. #4889 [ENHANCEMENT] AlertManager: Retrying AlertManager Get Requests (Get Alertmanager status, Get Alertmanager Receivers) on next replica on error #4840 [ENHANCEMENT] Querier/Ruler: Retry store-gateway in case of unexpected failure, instead of failing the query. #4532 #4839 [ENHANCEMENT] Ring: DoBatch prioritize 4xx errors when failing. #4783 [ENHANCEMENT] Cortex now built with Go 1.18. #4829 [ENHANCEMENT] Ingester: Prevent ingesters to become unhealthy during wall replay. #4847 [ENHANCEMENT] Compactor: Introduced visit marker file for blocks so blocks are under compaction will not be picked up by another compactor. #4805 [ENHANCEMENT] Distributor: Add label name to labelValueTooLongError. #4855 [ENHANCEMENT] Enhance traces with hostname information. #4898 [ENHANCEMENT] Improve the documentation around limits. #4905 [ENHANCEMENT] Distributor: cache user overrides to reduce lock contention. #4904 [BUGFIX] Storage/Bucket: Enable AWS SDK for go authentication for s3 to fix IMDSv1 authentication. #4897 [BUGFIX] Memberlist: Add join with no retrying when starting service. #4804 [BUGFIX] Ruler: Fix /ruler/rule_groups returns YAML with extra fields. #4767 [BUGFIX] Respecting -tracing.otel.sample-ratio configuration when enabling OpenTelemetry tracing with X-ray. #4862 [BUGFIX] QueryFrontend: fixed query_range requests when query has start equals to end. #4877 [BUGFIX] AlertManager: fixed issue introduced by #4495 where templates files were being deleted when using alertmanager local store. #4890 [BUGFIX] Ingester: fixed incorrect logging at the start of ingester block shipping logic. #4934 [BUGFIX] Storage/Bucket: fixed global mark missing on deletion. #4949 [BUGFIX] QueryFrontend/Querier: fixed regression added by #4863 where we stopped compressing the response between querier and query frontend. #4960 [BUGFIX] QueryFrontend/Querier: fixed fix response error to be ungzipped when status code is not 2xx. #4975 Known issues Configsdb: Ruler configs doesn’t work. Remove all configs from postgres database that have format Prometheus 1.x rule format before upgrading to v1.14.0 (see 5387) 1.13.0 2022-07-14 [CHANGE] Changed default for -ingester.min-ready-duration from 1 minute to 15 seconds. #4539 [CHANGE] query-frontend: Do not print anything in the logs of query-frontend if a in-progress query has been canceled (context canceled) to avoid spam. #4562 [CHANGE] Compactor block deletion mark migration, needed when upgrading from v1.7, is now disabled by default. #4597 [CHANGE] The status_code label on gRPC client metrics has changed from ‘200’ and ‘500’ to ‘2xx’, ‘5xx’, ‘4xx’, ‘cancel’ or ’error’. #4601 [CHANGE] Memberlist: changed probe interval from 1s to 5s and probe timeout from 500ms to 2s. #4601 [CHANGE] Fix incorrectly named cortex_cache_fetched_keys and cortex_cache_hits metrics. Renamed to cortex_cache_fetched_keys_total and cortex_cache_hits_total respectively. #4686 [CHANGE] Enable Thanos series limiter in store-gateway. #4702 [CHANGE] Distributor: Apply max_fetched_series_per_query limit for /series API. #4683 [CHANGE] Re-enable the proxy_url option for receiver configuration. #4741 [FEATURE] Ruler: Add external_labels option to tag all alerts with a given set of labels. #4499 [FEATURE] Compactor: Add -compactor.skip-blocks-with-out-of-order-chunks-enabled configuration to mark blocks containing index with out-of-order chunks for no compact instead of halting the compaction. #4707 [FEATURE] Querier/Query-Frontend: Add -querier.per-step-stats-enabled and -frontend.cache-queryable-samples-stats configurations to enable query sample statistics. #4708 [FEATURE] Add shuffle sharding for the compactor #4433 [FEATURE] Querier: Use streaming for ingester metadata APIs. #4725 [ENHANCEMENT] Update Go version to 1.17.8. #4602 #4604 #4658 [ENHANCEMENT] Keep track of discarded samples due to bad relabel configuration in cortex_discarded_samples_total. #4503 [ENHANCEMENT] Ruler: Add -ruler.disable-rule-group-label to disable the rule_group label on exported metrics. #4571 [ENHANCEMENT] Query federation: improve performance in MergeQueryable by memoizing labels. #4502 [ENHANCEMENT] Added new ring related config -ingester.readiness-check-ring-health when enabled the readiness probe will succeed only after all instances are ACTIVE and healthy in the ring, this is enabled by default. #4539 [ENHANCEMENT] Added new ring related config -distributor.excluded-zones when set this will exclude the comma-separated zones from the ring, default is “”. #4539 [ENHANCEMENT] Upgraded Docker base images to alpine:3.14. #4514 [ENHANCEMENT] Updated Prometheus to latest. Includes changes from prometheus#9239, adding 15 new functions. Multiple TSDB bugfixes prometheus#9438 \u0026 prometheus#9381. #4524 [ENHANCEMENT] Query Frontend: Add setting -frontend.forward-headers-list in frontend to configure the set of headers from the requests to be forwarded to downstream requests. #4486 [ENHANCEMENT] Blocks storage: Add -blocks-storage.azure.http.*, -alertmanager-storage.azure.http.*, and -ruler-storage.azure.http.* to configure the Azure storage client. #4581 [ENHANCEMENT] Optimise memberlist receive path when used as a backing store for rings with a large number of members. #4601 [ENHANCEMENT] Add length and limit to labelNameTooLongError and labelValueTooLongError #4595 [ENHANCEMENT] Add jitter to rejoinInterval. #4747 [ENHANCEMENT] Compactor: uploading blocks no compaction marks to the global location and introduce a new metric #4729 cortex_bucket_blocks_marked_for_no_compaction_count: Total number of blocks marked for no compaction in the bucket. [ENHANCEMENT] Querier: Reduce the number of series that are kept in memory while streaming from ingesters. #4745 [BUGFIX] AlertManager: remove stale template files. #4495 [BUGFIX] Distributor: fix bug in query-exemplar where some results would get dropped. #4583 [BUGFIX] Update Thanos dependency: compactor tracing support, azure blocks storage memory fix. #4585 [BUGFIX] Set appropriate Content-Type header for /services endpoint, which previously hard-coded text/plain. #4596 [BUGFIX] Querier: Disable query scheduler SRV DNS lookup, which removes noisy log messages about “failed DNS SRV record lookup”. #4601 [BUGFIX] Memberlist: fixed corrupted packets when sending compound messages with more than 255 messages or messages bigger than 64KB. #4601 [BUGFIX] Query Frontend: If ‘LogQueriesLongerThan’ is set to \u003c 0, log all queries as described in the docs. #4633 [BUGFIX] Distributor: update defaultReplicationStrategy to not fail with extend-write when a single instance is unhealthy. #4636 [BUGFIX] Distributor: Fix race condition on /series introduced by #4683. #4716 [BUGFIX] Ruler: Fixed leaking notifiers after users are removed #4718 [BUGFIX] Distributor: Fix a memory leak in distributor due to the cluster label. #4739 [BUGFIX] Memberlist: Avoid clock skew by limiting the timestamp accepted on gossip. #4750 [BUGFIX] Compactor: skip compaction if there is only 1 block available for shuffle-sharding compactor. #4756 [BUGFIX] Compactor: Fixes #4770 - an edge case in compactor with shulffle sharding where compaction stops when a tenant stops ingesting samples. #4771 [BUGFIX] Compactor: fix cortex_compactor_remaining_planned_compactions not set after plan generation for shuffle sharding compactor. #4772 1.11.0 2021-11-25 [CHANGE] Memberlist: Expose default configuration values to the command line options. Note that setting these explicitly to zero will no longer cause the default to be used. If the default is desired, then do set the option. The following are affected: #4276 -memberlist.stream-timeout -memberlist.retransmit-factor -memberlist.pull-push-interval -memberlist.gossip-interval -memberlist.gossip-nodes -memberlist.gossip-to-dead-nodes-time -memberlist.dead-node-reclaim-time [CHANGE] -querier.max-fetched-chunks-per-query previously applied to chunks from ingesters and store separately; now the two combined should not exceed the limit. #4260 [CHANGE] Memberlist: the metric memberlist_kv_store_value_bytes has been removed due to values no longer being stored in-memory as encoded bytes. #4345 [CHANGE] Some files and directories created by Cortex components on local disk now have stricter permissions, and are only readable by owner, but not group or others. #4394 [CHANGE] The metric cortex_deprecated_flags_inuse_total has been renamed to deprecated_flags_inuse_total as part of using grafana/dskit functionality. #4443 [FEATURE] Ruler: Add new -ruler.query-stats-enabled which when enabled will report the cortex_ruler_query_seconds_total as a per-user metric that tracks the sum of the wall time of executing queries in the ruler in seconds. #4317 [FEATURE] Query Frontend: Add cortex_query_fetched_series_total and cortex_query_fetched_chunks_bytes_total per-user counters to expose the number of series and bytes fetched as part of queries. These metrics can be enabled with the -frontend.query-stats-enabled flag (or its respective YAML config option query_stats_enabled). #4343 [FEATURE] AlertManager: Add support for SNS Receiver. #4382 [FEATURE] Distributor: Add label status to metric cortex_distributor_ingester_append_failures_total #4442 [FEATURE] Queries: Added present_over_time PromQL function, also some TSDB optimisations. #4505 [ENHANCEMENT] Add timeout for waiting on compactor to become ACTIVE in the ring. #4262 [ENHANCEMENT] Reduce memory used by streaming queries, particularly in ruler. #4341 [ENHANCEMENT] Ring: allow experimental configuration of disabling of heartbeat timeouts by setting the relevant configuration value to zero. Applies to the following: #4342 -distributor.ring.heartbeat-timeout -ring.heartbeat-timeout -ruler.ring.heartbeat-timeout -alertmanager.sharding-ring.heartbeat-timeout -compactor.ring.heartbeat-timeout -store-gateway.sharding-ring.heartbeat-timeout [ENHANCEMENT] Ring: allow heartbeats to be explicitly disabled by setting the interval to zero. This is considered experimental. This applies to the following configuration options: #4344 -distributor.ring.heartbeat-period -ingester.heartbeat-period -ruler.ring.heartbeat-period -alertmanager.sharding-ring.heartbeat-period -compactor.ring.heartbeat-period -store-gateway.sharding-ring.heartbeat-period [ENHANCEMENT] Memberlist: optimized receive path for processing ring state updates, to help reduce CPU utilization in large clusters. #4345 [ENHANCEMENT] Memberlist: expose configuration of memberlist packet compression via -memberlist.compression=enabled. #4346 [ENHANCEMENT] Update Go version to 1.16.6. #4362 [ENHANCEMENT] Updated Prometheus to include changes from prometheus/prometheus#9083. Now whenever /labels API calls include matchers, blocks store is queried for LabelNames with matchers instead of Series calls which was inefficient. #4380 [ENHANCEMENT] Querier: performance improvements in socket and memory handling. #4429 #4377 [ENHANCEMENT] Exemplars are now emitted for all gRPC calls and many operations tracked by histograms. #4462 [ENHANCEMENT] New options -server.http-listen-network and -server.grpc-listen-network allow binding as ’tcp4’ or ’tcp6’. #4462 [ENHANCEMENT] Rulers: Using shuffle sharding subring on GetRules API. #4466 [ENHANCEMENT] Support memcached auto-discovery via auto-discovery flag, introduced by thanos in https://github.com/thanos-io/thanos/pull/4487. Both AWS and Google Cloud memcached service support auto-discovery, which returns a list of nodes of the memcached cluster. #4412 [BUGFIX] Fixes a panic in the query-tee when comparing result. #4465 [BUGFIX] Frontend: Fixes @ modifier functions (start/end) when splitting queries by time. #4464 [BUGFIX] Compactor: compactor will no longer try to compact blocks that are already marked for deletion. Previously compactor would consider blocks marked for deletion within -compactor.deletion-delay / 2 period as eligible for compaction. #4328 [BUGFIX] HA Tracker: when cleaning up obsolete elected replicas from KV store, tracker didn’t update number of cluster per user correctly. #4336 [BUGFIX] Ruler: fixed counting of PromQL evaluation errors as user-errors when updating cortex_ruler_queries_failed_total. #4335 [BUGFIX] Ingester: When using block storage, prevent any reads or writes while the ingester is stopping. This will prevent accessing TSDB blocks once they have been already closed. #4304 [BUGFIX] Ingester: fixed ingester stuck on start up (LEAVING ring state) when -ingester.heartbeat-period=0 and -ingester.unregister-on-shutdown=false. #4366 [BUGFIX] Ingester: panic during shutdown while fetching batches from cache. #4397 [BUGFIX] Querier: After query-frontend restart, querier may have lower than configured concurrency. #4417 [BUGFIX] Memberlist: forward only changes, not entire original message. #4419 [BUGFIX] Memberlist: don’t accept old tombstones as incoming change, and don’t forward such messages to other gossip members. #4420 [BUGFIX] Querier: fixed panic when querying exemplars and using -distributor.shard-by-all-labels=false. #4473 [BUGFIX] Querier: honor querier minT,maxT if nil SelectHints are passed to Select(). #4413 [BUGFIX] Compactor: fixed panic while collecting Prometheus metrics. #4483 [BUGFIX] Update go-kit package to fix spurious log messages #4544 1.10.0 / 2021-08-03 [CHANGE] Prevent path traversal attack from users able to control the HTTP header X-Scope-OrgID. #4375 (CVE-2021-36157) Users only have control of the HTTP header when Cortex is not frontend by an auth proxy validating the tenant IDs [CHANGE] Enable strict JSON unmarshal for pkg/util/validation.Limits struct. The custom UnmarshalJSON() will now fail if the input has unknown fields. #4298 [CHANGE] Cortex chunks storage has been deprecated and it’s now in maintenance mode: all Cortex users are encouraged to migrate to the blocks storage. No new features will be added to the chunks storage. The default Cortex configuration still runs the chunks engine; please check out the blocks storage doc on how to configure Cortex to run with the blocks storage. #4268 [CHANGE] The example Kubernetes manifests (stored at k8s/) have been removed due to a lack of proper support and maintenance. #4268 [CHANGE] Querier / ruler: deprecated -store.query-chunk-limit CLI flag (and its respective YAML config option max_chunks_per_query) in favour of -querier.max-fetched-chunks-per-query (and its respective YAML config option max_fetched_chunks_per_query). The new limit specifies the maximum number of chunks that can be fetched in a single query from ingesters and long-term storage: the total number of actual fetched chunks could be 2x the limit, being independently applied when querying ingesters and long-term storage. #4125 [CHANGE] Alertmanager: allowed to configure the experimental receivers firewall on a per-tenant basis. The following CLI flags (and their respective YAML config options) have been changed and moved to the limits config section: #4143 -alertmanager.receivers-firewall.block.cidr-networks renamed to -alertmanager.receivers-firewall-block-cidr-networks -alertmanager.receivers-firewall.block.private-addresses renamed to -alertmanager.receivers-firewall-block-private-addresses [CHANGE] Change default value of -server.grpc.keepalive.min-time-between-pings from 5m to 10s and -server.grpc.keepalive.ping-without-stream-allowed to true. #4168 [CHANGE] Ingester: Change default value of -ingester.active-series-metrics-enabled to true. This incurs a small increase in memory usage, between 1.2% and 1.6% as measured on ingesters with 1.3M active series. #4257 [CHANGE] Dependency: update go-redis from v8.2.3 to v8.9.0. #4236 [FEATURE] Querier: Added new -querier.max-fetched-series-per-query flag. When Cortex is running with blocks storage, the max series per query limit is enforced in the querier and applies to unique series received from ingesters and store-gateway (long-term storage). #4179 [FEATURE] Querier/Ruler: Added new -querier.max-fetched-chunk-bytes-per-query flag. When Cortex is running with blocks storage, the max chunk bytes limit is enforced in the querier and ruler and limits the size of all aggregated chunks returned from ingesters and storage as bytes for a query. #4216 [FEATURE] Alertmanager: support negative matchers, time-based muting - upstream release notes. #4237 [FEATURE] Alertmanager: Added rate-limits to notifiers. Rate limits used by all integrations can be configured using -alertmanager.notification-rate-limit, while per-integration rate limits can be specified via -alertmanager.notification-rate-limit-per-integration parameter. Both shared and per-integration limits can be overwritten using overrides mechanism. These limits are applied on individual (per-tenant) alertmanagers. Rate-limited notifications are failed notifications. It is possible to monitor rate-limited notifications via new cortex_alertmanager_notification_rate_limited_total metric. #4135 #4163 [FEATURE] Alertmanager: Added -alertmanager.max-config-size-bytes limit to control size of configuration files that Cortex users can upload to Alertmanager via API. This limit is configurable per-tenant. #4201 [FEATURE] Alertmanager: Added -alertmanager.max-templates-count and -alertmanager.max-template-size-bytes options to control number and size of templates uploaded to Alertmanager via API. These limits are configurable per-tenant. #4223 [FEATURE] Added flag -debug.block-profile-rate to enable goroutine blocking events profiling. #4217 [FEATURE] Alertmanager: The experimental sharding feature is now considered complete. Detailed information about the configuration options can be found here for alertmanager and here for the alertmanager storage. To use the feature: #3925 #4020 #4021 #4031 #4084 #4110 #4126 #4127 #4141 #4146 #4161 #4162 #4222 Ensure that a remote storage backend is configured for Alertmanager to store state using -alertmanager-storage.backend, and flags related to the backend. Note that the local and configdb storage backends are not supported. Ensure that a ring store is configured using -alertmanager.sharding-ring.store, and set the flags relevant to the chosen store type. Enable the feature using -alertmanager.sharding-enabled. Note the prior addition of a new configuration option -alertmanager.persist-interval. This sets the interval between persisting the current alertmanager state (notification log and silences) to object storage. See the configuration file reference for more information. [ENHANCEMENT] Alertmanager: Cleanup persisted state objects from remote storage when a tenant configuration is deleted. #4167 [ENHANCEMENT] Storage: Added the ability to disable Open Census within GCS client (e.g -gcs.enable-opencensus=false). #4219 [ENHANCEMENT] Etcd: Added username and password to etcd config. #4205 [ENHANCEMENT] Alertmanager: introduced new metrics to monitor operation when using -alertmanager.sharding-enabled: #4149 cortex_alertmanager_state_fetch_replica_state_total cortex_alertmanager_state_fetch_replica_state_failed_total cortex_alertmanager_state_initial_sync_total cortex_alertmanager_state_initial_sync_completed_total cortex_alertmanager_state_initial_sync_duration_seconds cortex_alertmanager_state_persist_total cortex_alertmanager_state_persist_failed_total [ENHANCEMENT] Blocks storage: support ingesting exemplars and querying of exemplars. Enabled by setting new CLI flag -blocks-storage.tsdb.max-exemplars=\u003cn\u003e or config option blocks_storage.tsdb.max_exemplars to positive value. #4124 #4181 [ENHANCEMENT] Distributor: Added distributors ring status section in the admin page. #4151 [ENHANCEMENT] Added zone-awareness support to alertmanager for use when sharding is enabled. When zone-awareness is enabled, alerts will be replicated across availability zones. #4204 [ENHANCEMENT] Added tenant_ids tag to tracing spans #4186 [ENHANCEMENT] Ring, query-frontend: Avoid using automatic private IPs (APIPA) when discovering IP address from the interface during the registration of the instance in the ring, or by query-frontend when used with query-scheduler. APIPA still used as last resort with logging indicating usage. #4032 [ENHANCEMENT] Memberlist: introduced new metrics to aid troubleshooting tombstone convergence: #4231 memberlist_client_kv_store_value_tombstones memberlist_client_kv_store_value_tombstones_removed_total memberlist_client_messages_to_broadcast_dropped_total [ENHANCEMENT] Alertmanager: Added -alertmanager.max-dispatcher-aggregation-groups option to control max number of active dispatcher groups in Alertmanager (per tenant, also overridable). When the limit is reached, Dispatcher produces log message and increases cortex_alertmanager_dispatcher_aggregation_group_limit_reached_total metric. #4254 [ENHANCEMENT] Alertmanager: Added -alertmanager.max-alerts-count and -alertmanager.max-alerts-size-bytes to control max number of alerts and total size of alerts that a single user can have in Alertmanager’s memory. Adding more alerts will fail with a log message and incrementing cortex_alertmanager_alerts_insert_limited_total metric (per-user). These limits can be overrided by using per-tenant overrides. Current values are tracked in cortex_alertmanager_alerts_limiter_current_alerts and cortex_alertmanager_alerts_limiter_current_alerts_size_bytes metrics. #4253 [ENHANCEMENT] Store-gateway: added -store-gateway.sharding-ring.wait-stability-min-duration and -store-gateway.sharding-ring.wait-stability-max-duration support to store-gateway, to wait for ring stability at startup. #4271 [ENHANCEMENT] Ruler: added rule_group label to metrics cortex_prometheus_rule_group_iterations_total and cortex_prometheus_rule_group_iterations_missed_total. #4121 [ENHANCEMENT] Ruler: added new metrics for tracking total number of queries and push requests sent to ingester, as well as failed queries and push requests. Failures are only counted for internal errors, but not user-errors like limits or invalid query. This is in contrast to existing cortex_prometheus_rule_evaluation_failures_total, which is incremented also when query or samples appending fails due to user-errors. #4281 cortex_ruler_write_requests_total cortex_ruler_write_requests_failed_total cortex_ruler_queries_total cortex_ruler_queries_failed_total [ENHANCEMENT] Ingester: Added option -ingester.ignore-series-limit-for-metric-names with comma-separated list of metric names that will be ignored in max series per metric limit. #4302 [ENHANCEMENT] Added instrumentation to Redis client, with the following metrics: #3976 cortex_rediscache_request_duration_seconds [BUGFIX] Purger: fix Invalid null value in condition for column range caused by nil value in range for WriteBatch query. #4128 [BUGFIX] Ingester: fixed infrequent panic caused by a race condition between TSDB mmap-ed head chunks truncation and queries. #4176 [BUGFIX] Alertmanager: fix Alertmanager status page if clustering via gossip is disabled or sharding is enabled. #4184 [BUGFIX] Ruler: fix /ruler/rule_groups endpoint doesn’t work when used with object store. #4182 [BUGFIX] Ruler: Honor the evaluation delay for the ALERTS and ALERTS_FOR_STATE series. #4227 [BUGFIX] Make multiple Get requests instead of MGet on Redis Cluster. #4056 [BUGFIX] Ingester: fix issue where runtime limits erroneously override default limits. #4246 [BUGFIX] Ruler: fix startup in single-binary mode when the new ruler_storage is used. #4252 [BUGFIX] Querier: fix queries failing with “at least 1 healthy replica required, could only find 0” error right after scaling up store-gateways until they’re ACTIVE in the ring. #4263 [BUGFIX] Store-gateway: when blocks sharding is enabled, do not load all blocks in each store-gateway in case of a cold startup, but load only blocks owned by the store-gateway replica. #4271 [BUGFIX] Memberlist: fix to setting the default configuration value for -memberlist.retransmit-factor when not provided. This should improve propagation delay of the ring state (including, but not limited to, tombstones). Note that if the configuration is already explicitly given, this fix has no effect. #4269 [BUGFIX] Querier: Fix issue where samples in a chunk might get skipped by batch iterator. #4218 Blocksconvert [ENHANCEMENT] Scanner: add support for DynamoDB (v9 schema only). #3828 [ENHANCEMENT] Add Cassandra support. #3795 [ENHANCEMENT] Scanner: retry failed uploads. #4188 1.9.0 / 2021-05-14 [CHANGE] Alertmanager now removes local files after Alertmanager is no longer running for removed or resharded user. #3910 [CHANGE] Alertmanager now stores local files in per-tenant folders. Files stored by Alertmanager previously are migrated to new hierarchy. Support for this migration will be removed in Cortex 1.11. #3910 [CHANGE] Ruler: deprecated -ruler.storage.* CLI flags (and their respective YAML config options) in favour of -ruler-storage.*. The deprecated config will be removed in Cortex 1.11. #3945 [CHANGE] Alertmanager: deprecated -alertmanager.storage.* CLI flags (and their respective YAML config options) in favour of -alertmanager-storage.*. This change doesn’t apply to alertmanager.storage.path and alertmanager.storage.retention. The deprecated config will be removed in Cortex 1.11. #4002 [CHANGE] Alertmanager: removed -cluster. CLI flags deprecated in Cortex 1.7. The new config options to use are: #3946 -alertmanager.cluster.listen-address instead of -cluster.listen-address -alertmanager.cluster.advertise-address instead of -cluster.advertise-address -alertmanager.cluster.peers instead of -cluster.peer -alertmanager.cluster.peer-timeout instead of -cluster.peer-timeout [CHANGE] Blocks storage: removed the config option -blocks-storage.bucket-store.index-cache.postings-compression-enabled, which was deprecated in Cortex 1.6. Postings compression is always enabled. #4101 [CHANGE] Querier: removed the config option -store.max-look-back-period, which was deprecated in Cortex 1.6 and was used only by the chunks storage. You should use -querier.max-query-lookback instead. #4101 [CHANGE] Query Frontend: removed the config option -querier.compress-http-responses, which was deprecated in Cortex 1.6. You should use-api.response-compression-enabled instead. #4101 [CHANGE] Runtime-config / overrides: removed the config options -limits.per-user-override-config (use -runtime-config.file) and -limits.per-user-override-period (use -runtime-config.reload-period), both deprecated since Cortex 0.6.0. #4112 [CHANGE] Cortex now fails fast on startup if unable to connect to the ring backend. #4068 [FEATURE] The following features have been marked as stable: #4101 Shuffle-sharding Querier support for querying chunks and blocks store at the same time Tracking of active series and exporting them as metrics (-ingester.active-series-metrics-enabled and related flags) Blocks storage: lazy mmap of block indexes in the store-gateway (-blocks-storage.bucket-store.index-header-lazy-loading-enabled) Ingester: close idle TSDB and remove them from local disk (-blocks-storage.tsdb.close-idle-tsdb-timeout) [FEATURE] Memberlist: add TLS configuration options for the memberlist transport layer used by the gossip KV store. #4046 New flags added for memberlist communication: -memberlist.tls-enabled -memberlist.tls-cert-path -memberlist.tls-key-path -memberlist.tls-ca-path -memberlist.tls-server-name -memberlist.tls-insecure-skip-verify [FEATURE] Ruler: added local backend support to the ruler storage configuration under the -ruler-storage. flag prefix. #3932 [ENHANCEMENT] Upgraded Docker base images to alpine:3.13. #4042 [ENHANCEMENT] Blocks storage: reduce ingester memory by eliminating series reference cache. #3951 [ENHANCEMENT] Ruler: optimized \u003cprefix\u003e/api/v1/rules and \u003cprefix\u003e/api/v1/alerts when ruler sharding is enabled. #3916 [ENHANCEMENT] Ruler: added the following metrics when ruler sharding is enabled: #3916 cortex_ruler_clients cortex_ruler_client_request_duration_seconds [ENHANCEMENT] Alertmanager: Add API endpoint to list all tenant alertmanager configs: GET /multitenant_alertmanager/configs. #3529 [ENHANCEMENT] Ruler: Add API endpoint to list all tenant ruler rule groups: GET /ruler/rule_groups. #3529 [ENHANCEMENT] Query-frontend/scheduler: added querier forget delay (-query-frontend.querier-forget-delay and -query-scheduler.querier-forget-delay) to mitigate the blast radius in the event queriers crash because of a repeatedly sent “query of death” when shuffle-sharding is enabled. #3901 [ENHANCEMENT] Query-frontend: reduced memory allocations when serializing query response. #3964 [ENHANCEMENT] Querier / ruler: some optimizations to PromQL query engine. #3934 #3989 [ENHANCEMENT] Ingester: reduce CPU and memory when an high number of errors are returned by the ingester on the write path with the blocks storage. #3969 #3971 #3973 [ENHANCEMENT] Distributor: reduce CPU and memory when an high number of errors are returned by the distributor on the write path. #3990 [ENHANCEMENT] Put metric before label value in the “label value too long” error message. #4018 [ENHANCEMENT] Allow use of y|w|d suffixes for duration related limits and per-tenant limits. #4044 [ENHANCEMENT] Query-frontend: Small optimization on top of PR #3968 to avoid unnecessary Extents merging. #4026 [ENHANCEMENT] Add a metric cortex_compactor_compaction_interval_seconds for the compaction interval config value. #4040 [ENHANCEMENT] Ingester: added following per-ingester (instance) experimental limits: max number of series in memory (-ingester.instance-limits.max-series), max number of users in memory (-ingester.instance-limits.max-tenants), max ingestion rate (-ingester.instance-limits.max-ingestion-rate), and max inflight requests (-ingester.instance-limits.max-inflight-push-requests). These limits are only used when using blocks storage. Limits can also be configured using runtime-config feature, and current values are exported as cortex_ingester_instance_limits metric. #3992. [ENHANCEMENT] Cortex is now built with Go 1.16. #4062 [ENHANCEMENT] Distributor: added per-distributor experimental limits: max number of inflight requests (-distributor.instance-limits.max-inflight-push-requests) and max ingestion rate in samples/sec (-distributor.instance-limits.max-ingestion-rate). If not set, these two are unlimited. Also added metrics to expose current values (cortex_distributor_inflight_push_requests, cortex_distributor_ingestion_rate_samples_per_second) as well as limits (cortex_distributor_instance_limits with various limit label values). #4071 [ENHANCEMENT] Ruler: Added -ruler.enabled-tenants and -ruler.disabled-tenants to explicitly enable or disable rules processing for specific tenants. #4074 [ENHANCEMENT] Block Storage Ingester: /flush now accepts two new parameters: tenant to specify tenant to flush and wait=true to make call synchronous. Multiple tenants can be specified by repeating tenant parameter. If no tenant is specified, all tenants are flushed, as before. #4073 [ENHANCEMENT] Alertmanager: validate configured -alertmanager.web.external-url and fail if ends with /. #4081 [ENHANCEMENT] Alertmanager: added -alertmanager.receivers-firewall.block.cidr-networks and -alertmanager.receivers-firewall.block.private-addresses to block specific network addresses in HTTP-based Alertmanager receiver integrations. #4085 [ENHANCEMENT] Allow configuration of Cassandra’s host selection policy. #4069 [ENHANCEMENT] Store-gateway: retry syncing blocks if a per-tenant sync fails. #3975 #4088 [ENHANCEMENT] Add metric cortex_tcp_connections exposing the current number of accepted TCP connections. #4099 [ENHANCEMENT] Querier: Allow federated queries to run concurrently. #4065 [ENHANCEMENT] Label Values API call now supports match[] parameter when querying blocks on storage (assuming -querier.query-store-for-labels-enabled is enabled). #4133 [BUGFIX] Ruler-API: fix bug where /api/v1/rules/\u003cnamespace\u003e/\u003cgroup_name\u003e endpoint return 400 instead of 404. #4013 [BUGFIX] Distributor: reverted changes done to rate limiting in #3825. #3948 [BUGFIX] Ingester: Fix race condition when opening and closing tsdb concurrently. #3959 [BUGFIX] Querier: streamline tracing spans. #3924 [BUGFIX] Ruler Storage: ignore objects with empty namespace or group in the name. #3999 [BUGFIX] Distributor: fix issue causing distributors to not extend the replication set because of failing instances when zone-aware replication is enabled. #3977 [BUGFIX] Query-frontend: Fix issue where cached entry size keeps increasing when making tiny query repeatedly. #3968 [BUGFIX] Compactor: -compactor.blocks-retention-period now supports weeks (w) and years (y). #4027 [BUGFIX] Querier: returning 422 (instead of 500) when query hits max_chunks_per_query limit with block storage, when the limit is hit in the store-gateway. #3937 [BUGFIX] Ruler: Rule group limit enforcement should now allow the same number of rules in a group as the limit. #3616 [BUGFIX] Frontend, Query-scheduler: allow querier to notify about shutdown without providing any authentication. #4066 [BUGFIX] Querier: fixed race condition causing queries to fail right after querier startup with the “empty ring” error. #4068 [BUGFIX] Compactor: Increment cortex_compactor_runs_failed_total if compactor failed compact a single tenant. #4094 [BUGFIX] Tracing: hot fix to avoid the Jaeger tracing client to indefinitely block the Cortex process shutdown in case the HTTP connection to the tracing backend is blocked. #4134 [BUGFIX] Forward proper EndsAt from ruler to Alertmanager inline with Prometheus behaviour. #4017 [BUGFIX] Querier: support filtering LabelValues with matchers when using tenant federation. #4277 Blocksconvert [ENHANCEMENT] Builder: add -builder.timestamp-tolerance option which may reduce block size by rounding timestamps to make difference whole seconds. #3891 1.8.1 / 2021-04-27 [CHANGE] Fix for CVE-2021-31232: Local file disclosure vulnerability when -experimental.alertmanager.enable-api is used. The HTTP basic auth password_file can be used as an attack vector to send any file content via a webhook. The alertmanager templates can be used as an attack vector to send any file content because the alertmanager can load any text file specified in the templates list. 1.8.0 / 2021-03-24 [CHANGE] Alertmanager: Don’t expose cluster information to tenants via the /alertmanager/api/v1/status API endpoint when operating with clustering enabled. #3903 [CHANGE] Ingester: don’t update internal “last updated” timestamp of TSDB if tenant only sends invalid samples. This affects how “idle” time is computed. #3727 [CHANGE] Require explicit flag -\u003cprefix\u003e.tls-enabled to enable TLS in GRPC clients. Previously it was enough to specify a TLS flag to enable TLS validation. #3156 [CHANGE] Query-frontend: removed -querier.split-queries-by-day (deprecated in Cortex 0.4.0). Please use -querier.split-queries-by-interval instead. #3813 [CHANGE] Store-gateway: the chunks pool controlled by -blocks-storage.bucket-store.max-chunk-pool-bytes is now shared across all tenants. #3830 [CHANGE] Ingester: return error code 400 instead of 429 when per-user/per-tenant series/metadata limits are reached. #3833 [CHANGE] Compactor: add reason label to cortex_compactor_blocks_marked_for_deletion_total metric. Source blocks marked for deletion by compactor are labelled as compaction, while blocks passing the retention period are labelled as retention. #3879 [CHANGE] Alertmanager: the DELETE /api/v1/alerts is now idempotent. No error is returned if the alertmanager config doesn’t exist. #3888 [FEATURE] Experimental Ruler Storage: Add a separate set of configuration options to configure the ruler storage backend under the -ruler-storage. flag prefix. All blocks storage bucket clients and the config service are currently supported. Clients using this implementation will only be enabled if the existing -ruler.storage flags are left unset. #3805 #3864 [FEATURE] Experimental Alertmanager Storage: Add a separate set of configuration options to configure the alertmanager storage backend under the -alertmanager-storage. flag prefix. All blocks storage bucket clients and the config service are currently supported. Clients using this implementation will only be enabled if the existing -alertmanager.storage flags are left unset. #3888 [FEATURE] Adds support to S3 server-side encryption using KMS. The S3 server-side encryption config can be overridden on a per-tenant basis for the blocks storage, ruler and alertmanager. Deprecated -\u003cprefix\u003e.s3.sse-encryption, please use the following CLI flags that have been added. #3651 #3810 #3811 #3870 #3886 #3906 -\u003cprefix\u003e.s3.sse.type -\u003cprefix\u003e.s3.sse.kms-key-id -\u003cprefix\u003e.s3.sse.kms-encryption-context [FEATURE] Querier: Enable @ \u003ctimestamp\u003e modifier in PromQL using the new -querier.at-modifier-enabled flag. #3744 [FEATURE] Overrides Exporter: Add overrides-exporter module for exposing per-tenant resource limit overrides as metrics. It is not included in all target (single-binary mode), and must be explicitly enabled. #3785 [FEATURE] Experimental thanosconvert: introduce an experimental tool thanosconvert to migrate Thanos block metadata to Cortex metadata. #3770 [FEATURE] Alertmanager: It now shards the /api/v1/alerts API using the ring when sharding is enabled. #3671 Added -alertmanager.max-recv-msg-size (defaults to 16M) to limit the size of HTTP request body handled by the alertmanager. New flags added for communication between alertmanagers: -alertmanager.max-recv-msg-size -alertmanager.alertmanager-client.remote-timeout -alertmanager.alertmanager-client.tls-enabled -alertmanager.alertmanager-client.tls-cert-path -alertmanager.alertmanager-client.tls-key-path -alertmanager.alertmanager-client.tls-ca-path -alertmanager.alertmanager-client.tls-server-name -alertmanager.alertmanager-client.tls-insecure-skip-verify [FEATURE] Compactor: added blocks storage per-tenant retention support. This is configured via -compactor.retention-period, and can be overridden on a per-tenant basis. #3879 [ENHANCEMENT] Queries: Instrument queries that were discarded due to the configured max_outstanding_requests_per_tenant. #3894 cortex_query_frontend_discarded_requests_total cortex_query_scheduler_discarded_requests_total [ENHANCEMENT] Ruler: Add TLS and explicit basis authentication configuration options for the HTTP client the ruler uses to communicate with the alertmanager. #3752 -ruler.alertmanager-client.basic-auth-username: Configure the basic authentication username used by the client. Takes precedent over a URL configured username. -ruler.alertmanager-client.basic-auth-password: Configure the basic authentication password used by the client. Takes precedent over a URL configured password. -ruler.alertmanager-client.tls-ca-path: File path to the CA file. -ruler.alertmanager-client.tls-cert-path: File path to the TLS certificate. -ruler.alertmanager-client.tls-insecure-skip-verify: Boolean to disable verifying the certificate. -ruler.alertmanager-client.tls-key-path: File path to the TLS key certificate. -ruler.alertmanager-client.tls-server-name: Expected name on the TLS certificate. [ENHANCEMENT] Ingester: exposed metric cortex_ingester_oldest_unshipped_block_timestamp_seconds, tracking the unix timestamp of the oldest TSDB block not shipped to the storage yet. #3705 [ENHANCEMENT] Prometheus upgraded. #3739 #3806 Avoid unnecessary runtime.GC() during compactions. Prevent compaction loop in TSDB on data gap. [ENHANCEMENT] Query-Frontend now returns server side performance metrics using Server-Timing header when query stats is enabled. #3685 [ENHANCEMENT] Runtime Config: Add a mode query parameter for the runtime config endpoint. /runtime_config?mode=diff now shows the YAML runtime configuration with all values that differ from the defaults. #3700 [ENHANCEMENT] Distributor: Enable downstream projects to wrap distributor push function and access the deserialized write requests before/after they are pushed. #3755 [ENHANCEMENT] Add flag -\u003cprefix\u003e.tls-server-name to require a specific server name instead of the hostname on the certificate. #3156 [ENHANCEMENT] Alertmanager: Remove a tenant’s alertmanager instead of pausing it as we determine it is no longer needed. #3722 [ENHANCEMENT] Blocks storage: added more configuration options to S3 client. #3775 -blocks-storage.s3.tls-handshake-timeout: Maximum time to wait for a TLS handshake. 0 means no limit. -blocks-storage.s3.expect-continue-timeout: The time to wait for a server’s first response headers after fully writing the request headers if the request has an Expect header. 0 to send the request body immediately. -blocks-storage.s3.max-idle-connections: Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit. -blocks-storage.s3.max-idle-connections-per-host: Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. -blocks-storage.s3.max-connections-per-host: Maximum number of connections per host. 0 means no limit. [ENHANCEMENT] Ingester: when tenant’s TSDB is closed, Ingester now removes pushed metrics-metadata from memory, and removes metadata (cortex_ingester_memory_metadata, cortex_ingester_memory_metadata_created_total, cortex_ingester_memory_metadata_removed_total) and validation metrics (cortex_discarded_samples_total, cortex_discarded_metadata_total). #3782 [ENHANCEMENT] Distributor: cleanup metrics for inactive tenants. #3784 [ENHANCEMENT] Ingester: Have ingester to re-emit following TSDB metrics. #3800 cortex_ingester_tsdb_blocks_loaded cortex_ingester_tsdb_reloads_total cortex_ingester_tsdb_reloads_failures_total cortex_ingester_tsdb_symbol_table_size_bytes cortex_ingester_tsdb_storage_blocks_bytes cortex_ingester_tsdb_time_retentions_total [ENHANCEMENT] Querier: distribute workload across -store-gateway.sharding-ring.replication-factor store-gateway replicas when querying blocks and -store-gateway.sharding-enabled=true. #3824 [ENHANCEMENT] Distributor / HA Tracker: added cleanup of unused elected HA replicas from KV store. Added following metrics to monitor this process: #3809 cortex_ha_tracker_replicas_cleanup_started_total cortex_ha_tracker_replicas_cleanup_marked_for_deletion_total cortex_ha_tracker_replicas_cleanup_deleted_total cortex_ha_tracker_replicas_cleanup_delete_failed_total [ENHANCEMENT] Ruler now has new API endpoint /ruler/delete_tenant_config that can be used to delete all ruler groups for tenant. It is intended to be used by administrators who wish to clean up state after removed user. Note that this endpoint is enabled regardless of -experimental.ruler.enable-api. #3750 #3899 [ENHANCEMENT] Query-frontend, query-scheduler: cleanup metrics for inactive tenants. #3826 [ENHANCEMENT] Blocks storage: added -blocks-storage.s3.region support to S3 client configuration. #3811 [ENHANCEMENT] Distributor: Remove cached subrings for inactive users when using shuffle sharding. #3849 [ENHANCEMENT] Store-gateway: Reduced memory used to fetch chunks at query time. #3855 [ENHANCEMENT] Ingester: attempt to prevent idle compaction from happening in concurrent ingesters by introducing a 25% jitter to the configured idle timeout (-blocks-storage.tsdb.head-compaction-idle-timeout). #3850 [ENHANCEMENT] Compactor: cleanup local files for users that are no longer owned by compactor. #3851 [ENHANCEMENT] Store-gateway: close empty bucket stores, and delete leftover local files for tenants that no longer belong to store-gateway. #3853 [ENHANCEMENT] Store-gateway: added metrics to track partitioner behaviour. #3877 cortex_bucket_store_partitioner_requested_bytes_total cortex_bucket_store_partitioner_requested_ranges_total cortex_bucket_store_partitioner_expanded_bytes_total cortex_bucket_store_partitioner_expanded_ranges_total [ENHANCEMENT] Store-gateway: added metrics to monitor chunk buffer pool behaviour. #3880 cortex_bucket_store_chunk_pool_requested_bytes_total cortex_bucket_store_chunk_pool_returned_bytes_total [ENHANCEMENT] Alertmanager: load alertmanager configurations from object storage concurrently, and only load necessary configurations, speeding configuration synchronization process and executing fewer “GET object” operations to the storage when sharding is enabled. #3898 [ENHANCEMENT] Ingester (blocks storage): Ingester can now stream entire chunks instead of individual samples to the querier. At the moment this feature must be explicitly enabled either by using -ingester.stream-chunks-when-using-blocks flag or ingester_stream_chunks_when_using_blocks (boolean) field in runtime config file, but these configuration options are temporary and will be removed when feature is stable. #3889 [ENHANCEMENT] Alertmanager: New endpoint /multitenant_alertmanager/delete_tenant_config to delete configuration for tenant identified by X-Scope-OrgID header. This is an internal endpoint, available even if Alertmanager API is not enabled by using -experimental.alertmanager.enable-api. #3900 [ENHANCEMENT] MemCached: Add max_item_size support. #3929 [BUGFIX] Cortex: Fixed issue where fatal errors and various log messages where not logged. #3778 [BUGFIX] HA Tracker: don’t track as error in the cortex_kv_request_duration_seconds metric a CAS operation intentionally aborted. #3745 [BUGFIX] Querier / ruler: do not log “error removing stale clients” if the ring is empty. #3761 [BUGFIX] Store-gateway: fixed a panic caused by a race condition when the index-header lazy loading is enabled. #3775 #3789 [BUGFIX] Compactor: fixed “could not guess file size” log when uploading blocks deletion marks to the global location. #3807 [BUGFIX] Prevent panic at start if the http_prefix setting doesn’t have a valid value. #3796 [BUGFIX] Memberlist: fixed panic caused by race condition in armon/go-metrics used by memberlist client. #3725 [BUGFIX] Querier: returning 422 (instead of 500) when query hits max_chunks_per_query limit with block storage. #3895 [BUGFIX] Alertmanager: Ensure that experimental /api/v1/alerts endpoints work when -http.prefix is empty. #3905 [BUGFIX] Chunk store: fix panic in inverted index when deleted fingerprint is no longer in the index. #3543 1.7.1 / 2021-04-27 [CHANGE] Fix for CVE-2021-31232: Local file disclosure vulnerability when -experimental.alertmanager.enable-api is used. The HTTP basic auth password_file can be used as an attack vector to send any file content via a webhook. The alertmanager templates can be used as an attack vector to send any file content because the alertmanager can load any text file specified in the templates list. 1.7.0 / 2021-02-23 Note the blocks storage compactor runs a migration task at startup in this version, which can take many minutes and use a lot of RAM. Turn this off after first run.\n[CHANGE] FramedSnappy encoding support has been removed from Push and Remote Read APIs. This means Prometheus 1.6 support has been removed and the oldest Prometheus version supported in the remote write is 1.7. #3682 [CHANGE] Ruler: removed the flag -ruler.evaluation-delay-duration-deprecated which was deprecated in 1.4.0. Please use the ruler_evaluation_delay_duration per-tenant limit instead. #3694 [CHANGE] Removed the flags -\u003cprefix\u003e.grpc-use-gzip-compression which were deprecated in 1.3.0: #3694 -query-scheduler.grpc-client-config.grpc-use-gzip-compression: use -query-scheduler.grpc-client-config.grpc-compression instead -frontend.grpc-client-config.grpc-use-gzip-compression: use -frontend.grpc-client-config.grpc-compression instead -ruler.client.grpc-use-gzip-compression: use -ruler.client.grpc-compression instead -bigtable.grpc-use-gzip-compression: use -bigtable.grpc-compression instead -ingester.client.grpc-use-gzip-compression: use -ingester.client.grpc-compression instead -querier.frontend-client.grpc-use-gzip-compression: use -querier.frontend-client.grpc-compression instead [CHANGE] Querier: it’s not required to set -frontend.query-stats-enabled=true in the querier anymore to enable query statistics logging in the query-frontend. The flag is now required to be configured only in the query-frontend and it will be propagated to the queriers. #3595 #3695 [CHANGE] Blocks storage: compactor is now required when running a Cortex cluster with the blocks storage, because it also keeps the bucket index updated. #3583 [CHANGE] Blocks storage: block deletion marks are now stored in a per-tenant global markers/ location too, other than within the block location. The compactor, at startup, will copy deletion marks from the block location to the global location. This migration is required only once, so it can be safely disabled via -compactor.block-deletion-marks-migration-enabled=false after new compactor has successfully started at least once in the cluster. #3583 [CHANGE] OpenStack Swift: the default value for the -ruler.storage.swift.container-name and -swift.container-name config options has changed from cortex to empty string. If you were relying on the default value, please set it back to cortex. #3660 [CHANGE] HA Tracker: configured replica label is now verified against label value length limit (-validation.max-length-label-value). #3668 [CHANGE] Distributor: extend_writes field in YAML configuration has moved from lifecycler (inside ingester_config) to distributor_config. This doesn’t affect command line option -distributor.extend-writes, which stays the same. #3719 [CHANGE] Alertmanager: Deprecated -cluster. CLI flags in favor of their -alertmanager.cluster. equivalent. The deprecated flags (and their respective YAML config options) are: #3677 -cluster.listen-address in favor of -alertmanager.cluster.listen-address -cluster.advertise-address in favor of -alertmanager.cluster.advertise-address -cluster.peer in favor of -alertmanager.cluster.peers -cluster.peer-timeout in favor of -alertmanager.cluster.peer-timeout [CHANGE] Blocks storage: the default value of -blocks-storage.bucket-store.sync-interval has been changed from 5m to 15m. #3724 [FEATURE] Querier: Queries can be federated across multiple tenants. The tenants IDs involved need to be specified separated by a | character in the X-Scope-OrgID request header. This is an experimental feature, which can be enabled by setting -tenant-federation.enabled=true on all Cortex services. #3250 [FEATURE] Alertmanager: introduced the experimental option -alertmanager.sharding-enabled to shard tenants across multiple Alertmanager instances. This feature is still under heavy development and its usage is discouraged. The following new metrics are exported by the Alertmanager: #3664 cortex_alertmanager_ring_check_errors_total cortex_alertmanager_sync_configs_total cortex_alertmanager_sync_configs_failed_total cortex_alertmanager_tenants_discovered cortex_alertmanager_tenants_owned [ENHANCEMENT] Allow specifying JAEGER_ENDPOINT instead of sampling server or local agent port. #3682 [ENHANCEMENT] Blocks storage: introduced a per-tenant bucket index, periodically updated by the compactor, used to avoid full bucket scanning done by queriers, store-gateways and rulers. The bucket index is updated by the compactor during blocks cleanup, on every -compactor.cleanup-interval. #3553 #3555 #3561 #3583 #3625 #3711 #3715 [ENHANCEMENT] Blocks storage: introduced an option -blocks-storage.bucket-store.bucket-index.enabled to enable the usage of the bucket index in the querier, store-gateway and ruler. When enabled, the querier, store-gateway and ruler will use the bucket index to find a tenant’s blocks instead of running the periodic bucket scan. The following new metrics are exported by the querier and ruler: #3614 #3625 cortex_bucket_index_loads_total cortex_bucket_index_load_failures_total cortex_bucket_index_load_duration_seconds cortex_bucket_index_loaded [ENHANCEMENT] Compactor: exported the following metrics. #3583 #3625 cortex_bucket_blocks_count: Total number of blocks per tenant in the bucket. Includes blocks marked for deletion, but not partial blocks. cortex_bucket_blocks_marked_for_deletion_count: Total number of blocks per tenant marked for deletion in the bucket. cortex_bucket_blocks_partials_count: Total number of partial blocks. cortex_bucket_index_last_successful_update_timestamp_seconds: Timestamp of the last successful update of a tenant’s bucket index. [ENHANCEMENT] Ruler: Add cortex_prometheus_last_evaluation_samples to expose the number of samples generated by a rule group per tenant. #3582 [ENHANCEMENT] Memberlist: add status page (/memberlist) with available details about memberlist-based KV store and memberlist cluster. It’s also possible to view KV values in Go struct or JSON format, or download for inspection. #3575 [ENHANCEMENT] Memberlist: client can now keep a size-bounded buffer with sent and received messages and display them in the admin UI (/memberlist) for troubleshooting. #3581 #3602 [ENHANCEMENT] Blocks storage: added block index attributes caching support to metadata cache. The TTL can be configured via -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl. #3629 [ENHANCEMENT] Alertmanager: Add support for Azure blob storage. #3634 [ENHANCEMENT] Compactor: tenants marked for deletion will now be fully cleaned up after some delay since deletion of last block. Cleanup includes removal of remaining marker files (including tenant deletion mark file) and files under debug/metas. #3613 [ENHANCEMENT] Compactor: retry compaction of a single tenant on failure instead of re-running compaction for all tenants. #3627 [ENHANCEMENT] Querier: Implement result caching for tenant query federation. #3640 [ENHANCEMENT] API: Add a mode query parameter for the config endpoint: #3645 /config?mode=diff: Shows the YAML configuration with all values that differ from the defaults. /config?mode=defaults: Shows the YAML configuration with all the default values. [ENHANCEMENT] OpenStack Swift: added the following config options to OpenStack Swift backend client: #3660 Chunks storage: -swift.auth-version, -swift.max-retries, -swift.connect-timeout, -swift.request-timeout. Blocks storage: -blocks-storage.swift.auth-version, -blocks-storage.swift.max-retries, -blocks-storage.swift.connect-timeout, -blocks-storage.swift.request-timeout. Ruler: -ruler.storage.swift.auth-version, -ruler.storage.swift.max-retries, -ruler.storage.swift.connect-timeout, -ruler.storage.swift.request-timeout. [ENHANCEMENT] Disabled in-memory shuffle-sharding subring cache in the store-gateway, ruler and compactor. This should reduce the memory utilisation in these services when shuffle-sharding is enabled, without introducing a significantly increase CPU utilisation. #3601 [ENHANCEMENT] Shuffle sharding: optimised subring generation used by shuffle sharding. #3601 [ENHANCEMENT] New /runtime_config endpoint that returns the defined runtime configuration in YAML format. The returned configuration includes overrides. #3639 [ENHANCEMENT] Query-frontend: included the parameter name failed to validate in HTTP 400 message. #3703 [ENHANCEMENT] Fail to startup Cortex if provided runtime config is invalid. #3707 [ENHANCEMENT] Alertmanager: Add flags to customize the cluster configuration: #3667 -alertmanager.cluster.gossip-interval: The interval between sending gossip messages. By lowering this value (more frequent) gossip messages are propagated across cluster more quickly at the expense of increased bandwidth usage. -alertmanager.cluster.push-pull-interval: The interval between gossip state syncs. Setting this interval lower (more frequent) will increase convergence speeds across larger clusters at the expense of increased bandwidth usage. [ENHANCEMENT] Distributor: change the error message returned when a received series has too many label values. The new message format has the series at the end and this plays better with Prometheus logs truncation. #3718 From: sample for '\u003cseries\u003e' has \u003cvalue\u003e label names; limit \u003cvalue\u003e To: series has too many labels (actual: \u003cvalue\u003e, limit: \u003cvalue\u003e) series: '\u003cseries\u003e' [ENHANCEMENT] Improve bucket index loader to handle edge case where new tenant has not had blocks uploaded to storage yet. #3717 [BUGFIX] Allow -querier.max-query-lookback use y|w|d suffix like deprecated -store.max-look-back-period. #3598 [BUGFIX] Memberlist: Entry in the ring should now not appear again after using “Forget” feature (unless it’s still heartbeating). #3603 [BUGFIX] Ingester: do not close idle TSDBs while blocks shipping is in progress. #3630 #3632 [BUGFIX] Ingester: correctly update cortex_ingester_memory_users and cortex_ingester_active_series when a tenant’s idle TSDB is closed, when running Cortex with the blocks storage. #3646 [BUGFIX] Querier: fix default value incorrectly overriding -querier.frontend-address in single-binary mode. #3650 [BUGFIX] Compactor: delete deletion-mark.json at last when deleting a block in order to not leave partial blocks without deletion mark in the bucket if the compactor is interrupted while deleting a block. #3660 [BUGFIX] Blocks storage: do not cleanup a partially uploaded block when meta.json upload fails. Despite failure to upload meta.json, this file may in some cases still appear in the bucket later. By skipping early cleanup, we avoid having corrupted blocks in the storage. #3660 [BUGFIX] Alertmanager: disable access to /alertmanager/metrics (which exposes all Cortex metrics), /alertmanager/-/reload and /alertmanager/debug/*, which were available to any authenticated user with enabled AlertManager. #3678 [BUGFIX] Query-Frontend: avoid creating many small sub-queries by discarding cache extents under 5 minutes #3653 [BUGFIX] Ruler: Ensure the stale markers generated for evaluated rules respect the configured -ruler.evaluation-delay-duration. This will avoid issues with samples with NaN be persisted with timestamps set ahead of the next rule evaluation. #3687 [BUGFIX] Alertmanager: don’t serve HTTP requests until Alertmanager has fully started. Serving HTTP requests earlier may result in loss of configuration for the user. #3679 [BUGFIX] Do not log “failed to load config” if runtime config file is empty. #3706 [BUGFIX] Do not allow to use a runtime config file containing multiple YAML documents. #3706 [BUGFIX] HA Tracker: don’t track as error in the cortex_kv_request_duration_seconds metric a CAS operation intentionally aborted. #3745 1.6.0 / 2020-12-29 [CHANGE] Query Frontend: deprecate -querier.compress-http-responses in favour of -api.response-compression-enabled. #3544 [CHANGE] Querier: deprecated -store.max-look-back-period. You should use -querier.max-query-lookback instead. #3452 [CHANGE] Blocks storage: increased -blocks-storage.bucket-store.chunks-cache.attributes-ttl default from 24h to 168h (1 week). #3528 [CHANGE] Blocks storage: the config option -blocks-storage.bucket-store.index-cache.postings-compression-enabled has been deprecated and postings compression is always enabled. #3538 [CHANGE] Ruler: gRPC message size default limits on the Ruler-client side have changed: #3523 limit for outgoing gRPC messages has changed from 2147483647 to 16777216 bytes limit for incoming gRPC messages has changed from 4194304 to 104857600 bytes [FEATURE] Distributor/Ingester: Provide ability to not overflow writes in the presence of a leaving or unhealthy ingester. This allows for more efficient ingester rolling restarts. #3305 [FEATURE] Query-frontend: introduced query statistics logged in the query-frontend when enabled via -frontend.query-stats-enabled=true. When enabled, the metric cortex_query_seconds_total is tracked, counting the sum of the wall time spent across all queriers while running queries (on a per-tenant basis). The metrics cortex_request_duration_seconds and cortex_query_seconds_total are different: the first one tracks the request duration (eg. HTTP request from the client), while the latter tracks the sum of the wall time on all queriers involved executing the query. #3539 [ENHANCEMENT] API: Add GZIP HTTP compression to the API responses. Compression can be enabled via -api.response-compression-enabled. #3536 [ENHANCEMENT] Added zone-awareness support on queries. When zone-awareness is enabled, queries will still succeed if all ingesters in a single zone will fail. #3414 [ENHANCEMENT] Blocks storage ingester: exported more TSDB-related metrics. #3412 cortex_ingester_tsdb_wal_corruptions_total cortex_ingester_tsdb_head_truncations_failed_total cortex_ingester_tsdb_head_truncations_total cortex_ingester_tsdb_head_gc_duration_seconds [ENHANCEMENT] Enforced keepalive on all gRPC clients used for inter-service communication. #3431 [ENHANCEMENT] Added cortex_alertmanager_config_hash metric to expose hash of Alertmanager Config loaded per user. #3388 [ENHANCEMENT] Query-Frontend / Query-Scheduler: New component called “Query-Scheduler” has been introduced. Query-Scheduler is simply a queue of requests, moved outside of Query-Frontend. This allows Query-Frontend to be scaled separately from number of queues. To make Query-Frontend and Querier use Query-Scheduler, they need to be started with -frontend.scheduler-address and -querier.scheduler-address options respectively. #3374 #3471 [ENHANCEMENT] Query-frontend / Querier / Ruler: added -querier.max-query-lookback to limit how long back data (series and metadata) can be queried. This setting can be overridden on a per-tenant basis and is enforced in the query-frontend, querier and ruler. #3452 #3458 [ENHANCEMENT] Querier: added -querier.query-store-for-labels-enabled to query store for label names, label values and series APIs. Only works with blocks storage engine. #3461 #3520 [ENHANCEMENT] Ingester: exposed -blocks-storage.tsdb.wal-segment-size-bytes config option to customise the TSDB WAL segment max size. #3476 [ENHANCEMENT] Compactor: concurrently run blocks cleaner for multiple tenants. Concurrency can be configured via -compactor.cleanup-concurrency. #3483 [ENHANCEMENT] Compactor: shuffle tenants before running compaction. #3483 [ENHANCEMENT] Compactor: wait for a stable ring at startup, when sharding is enabled. #3484 [ENHANCEMENT] Store-gateway: added -blocks-storage.bucket-store.index-header-lazy-loading-enabled to enable index-header lazy loading (experimental). When enabled, index-headers will be mmap-ed only once required by a query and will be automatically released after -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout time of inactivity. #3498 [ENHANCEMENT] Alertmanager: added metrics cortex_alertmanager_notification_requests_total and cortex_alertmanager_notification_requests_failed_total. #3518 [ENHANCEMENT] Ingester: added -blocks-storage.tsdb.head-chunks-write-buffer-size-bytes to fine-tune the TSDB head chunks write buffer size when running Cortex blocks storage. #3518 [ENHANCEMENT] /metrics now supports OpenMetrics output. HTTP and gRPC servers metrics can now include exemplars. #3524 [ENHANCEMENT] Expose gRPC keepalive policy options by gRPC server. #3524 [ENHANCEMENT] Blocks storage: enabled caching of meta.json attributes, configurable via -blocks-storage.bucket-store.metadata-cache.metafile-attributes-ttl. #3528 [ENHANCEMENT] Compactor: added a config validation check to fail fast if the compactor has been configured invalid block range periods (each period is expected to be a multiple of the previous one). #3534 [ENHANCEMENT] Blocks storage: concurrently fetch deletion marks from object storage. #3538 [ENHANCEMENT] Blocks storage ingester: ingester can now close idle TSDB and delete local data. #3491 #3552 [ENHANCEMENT] Blocks storage: add option to use V2 signatures for S3 authentication. #3540 [ENHANCEMENT] Exported process metrics to monitor the number of memory map areas allocated. #3537 process_memory_map_areas process_memory_map_areas_limit [ENHANCEMENT] Ruler: Expose gRPC client options. #3523 [ENHANCEMENT] Compactor: added metrics to track on-going compaction. #3535 cortex_compactor_tenants_discovered cortex_compactor_tenants_skipped cortex_compactor_tenants_processing_succeeded cortex_compactor_tenants_processing_failed [ENHANCEMENT] Added new experimental API endpoints: POST /purger/delete_tenant and GET /purger/delete_tenant_status for deleting all tenant data. Only works with blocks storage. Compactor removes blocks that belong to user marked for deletion. #3549 #3558 [ENHANCEMENT] Chunks storage: add option to use V2 signatures for S3 authentication. #3560 [ENHANCEMENT] HA Tracker: Added new limit ha_max_clusters to set the max number of clusters tracked for single user. This limit is disabled by default. #3668 [BUGFIX] Query-Frontend: cortex_query_seconds_total now return seconds not nanoseconds. #3589 [BUGFIX] Blocks storage ingester: fixed some cases leading to a TSDB WAL corruption after a partial write to disk. #3423 [BUGFIX] Blocks storage: Fix the race between ingestion and /flush call resulting in overlapping blocks. #3422 [BUGFIX] Querier: fixed -querier.max-query-into-future which wasn’t correctly enforced on range queries. #3452 [BUGFIX] Fixed float64 precision stability when aggregating metrics before exposing them. This could have lead to false counters resets when querying some metrics exposed by Cortex. #3506 [BUGFIX] Querier: the meta.json sync concurrency done when running Cortex with the blocks storage is now controlled by -blocks-storage.bucket-store.meta-sync-concurrency instead of the incorrect -blocks-storage.bucket-store.block-sync-concurrency (default values are the same). #3531 [BUGFIX] Querier: fixed initialization order of querier module when using blocks storage. It now (again) waits until blocks have been synchronized. #3551 Blocksconvert [ENHANCEMENT] Scheduler: ability to ignore users based on regexp, using -scheduler.ignore-users-regex flag. #3477 [ENHANCEMENT] Builder: Parallelize reading chunks in the final stage of building block. #3470 [ENHANCEMENT] Builder: remove duplicate label names from chunk. #3547 1.5.0 / 2020-11-09 Cortex [CHANGE] Blocks storage: update the default HTTP configuration values for the S3 client to the upstream Thanos default values. #3244 -blocks-storage.s3.http.idle-conn-timeout is set 90 seconds. -blocks-storage.s3.http.response-header-timeout is set to 2 minutes. [CHANGE] Improved shuffle sharding support in the write path. This work introduced some config changes: #3090 Introduced -distributor.sharding-strategy CLI flag (and its respective sharding_strategy YAML config option) to explicitly specify which sharding strategy should be used in the write path -experimental.distributor.user-subring-size flag renamed to -distributor.ingestion-tenant-shard-size user_subring_size limit YAML config option renamed to ingestion_tenant_shard_size [CHANGE] Dropped “blank Alertmanager configuration; using fallback” message from Info to Debug level. #3205 [CHANGE] Zone-awareness replication for time-series now should be explicitly enabled in the distributor via the -distributor.zone-awareness-enabled CLI flag (or its respective YAML config option). Before, zone-aware replication was implicitly enabled if a zone was set on ingesters. #3200 [CHANGE] Removed the deprecated CLI flag -config-yaml. You should use -schema-config-file instead. #3225 [CHANGE] Enforced the HTTP method required by some API endpoints which did (incorrectly) allow any method before that. #3228 GET / GET /config GET /debug/fgprof GET /distributor/all_user_stats GET /distributor/ha_tracker GET /all_user_stats GET /ha-tracker GET /api/v1/user_stats GET /api/v1/chunks GET \u003clegacy-http-prefix\u003e/user_stats GET \u003clegacy-http-prefix\u003e/chunks GET /services GET /multitenant_alertmanager/status GET /status (alertmanager microservice) GET|POST /ingester/ring GET|POST /ring GET|POST /store-gateway/ring GET|POST /compactor/ring GET|POST /ingester/flush GET|POST /ingester/shutdown GET|POST /flush GET|POST /shutdown GET|POST /ruler/ring POST /api/v1/push POST \u003clegacy-http-prefix\u003e/push POST /push POST /ingester/push [CHANGE] Renamed CLI flags to configure the network interface names from which automatically detect the instance IP. #3295 -compactor.ring.instance-interface renamed to -compactor.ring.instance-interface-names -store-gateway.sharding-ring.instance-interface renamed to -store-gateway.sharding-ring.instance-interface-names -distributor.ring.instance-interface renamed to -distributor.ring.instance-interface-names -ruler.ring.instance-interface renamed to -ruler.ring.instance-interface-names [CHANGE] Renamed -\u003cprefix\u003e.redis.enable-tls CLI flag to -\u003cprefix\u003e.redis.tls-enabled, and its respective YAML config option from enable_tls to tls_enabled. #3298 [CHANGE] Increased default -\u003cprefix\u003e.redis.timeout from 100ms to 500ms. #3301 [CHANGE] cortex_alertmanager_config_invalid has been removed in favor of cortex_alertmanager_config_last_reload_successful. #3289 [CHANGE] Query-frontend: POST requests whose body size exceeds 10MiB will be rejected. The max body size can be customised via -frontend.max-body-size. #3276 [FEATURE] Shuffle sharding: added support for shuffle-sharding queriers in the query-frontend. When configured (-frontend.max-queriers-per-tenant globally, or using per-tenant limit max_queriers_per_tenant), each tenants’s requests will be handled by different set of queriers. #3113 #3257 [FEATURE] Shuffle sharding: added support for shuffle-sharding ingesters on the read path. When ingesters shuffle-sharding is enabled and -querier.shuffle-sharding-ingesters-lookback-period is set, queriers will fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since ’now - lookback period’. #3252 [FEATURE] Query-frontend: added compression config to support results cache with compression. #3217 [FEATURE] Add OpenStack Swift support to blocks storage. #3303 [FEATURE] Added support for applying Prometheus relabel configs on series received by the distributor. A metric_relabel_configs field has been added to the per-tenant limits configuration. #3329 [FEATURE] Support for Cassandra client SSL certificates. #3384 [ENHANCEMENT] Ruler: Introduces two new limits -ruler.max-rules-per-rule-group and -ruler.max-rule-groups-per-tenant to control the number of rules per rule group and the total number of rule groups for a given user. They are disabled by default. #3366 [ENHANCEMENT] Allow to specify multiple comma-separated Cortex services to -target CLI option (or its respective YAML config option). For example, -target=all,compactor can be used to start Cortex single-binary with compactor as well. #3275 [ENHANCEMENT] Expose additional HTTP configs for the S3 backend client. New flag are listed below: #3244 -blocks-storage.s3.http.idle-conn-timeout -blocks-storage.s3.http.response-header-timeout -blocks-storage.s3.http.insecure-skip-verify [ENHANCEMENT] Added cortex_query_frontend_connected_clients metric to show the number of workers currently connected to the frontend. #3207 [ENHANCEMENT] Shuffle sharding: improved shuffle sharding in the write path. Shuffle sharding now should be explicitly enabled via -distributor.sharding-strategy CLI flag (or its respective YAML config option) and guarantees stability, consistency, shuffling and balanced zone-awareness properties. #3090 #3214 [ENHANCEMENT] Ingester: added new metric cortex_ingester_active_series to track active series more accurately. Also added options to control whether active series tracking is enabled (-ingester.active-series-metrics-enabled, defaults to false), and how often this metric is updated (-ingester.active-series-metrics-update-period) and max idle time for series to be considered inactive (-ingester.active-series-metrics-idle-timeout). #3153 [ENHANCEMENT] Store-gateway: added zone-aware replication support to blocks replication in the store-gateway. #3200 [ENHANCEMENT] Store-gateway: exported new metrics. #3231 cortex_bucket_store_cached_series_fetch_duration_seconds cortex_bucket_store_cached_postings_fetch_duration_seconds cortex_bucket_stores_gate_queries_max [ENHANCEMENT] Added -version flag to Cortex. #3233 [ENHANCEMENT] Hash ring: added instance registered timestamp to the ring. #3248 [ENHANCEMENT] Reduce tail latency by smoothing out spikes in rate of chunk flush operations. #3191 [ENHANCEMENT] User Cortex as User Agent in http requests issued by Configs DB client. #3264 [ENHANCEMENT] Experimental Ruler API: Fetch rule groups from object storage in parallel. #3218 [ENHANCEMENT] Chunks GCS object storage client uses the fields selector to limit the payload size when listing objects in the bucket. #3218 #3292 [ENHANCEMENT] Added shuffle sharding support to ruler. Added new metric cortex_ruler_sync_rules_total. #3235 [ENHANCEMENT] Return an explicit error when the store-gateway is explicitly requested without a blocks storage engine. #3287 [ENHANCEMENT] Ruler: only load rules that belong to the ruler. Improves rules syncing performances when ruler sharding is enabled. #3269 [ENHANCEMENT] Added -\u003cprefix\u003e.redis.tls-insecure-skip-verify flag. #3298 [ENHANCEMENT] Added cortex_alertmanager_config_last_reload_successful_seconds metric to show timestamp of last successful AM config reload. #3289 [ENHANCEMENT] Blocks storage: reduced number of bucket listing operations to list block content (applies to newly created blocks only). #3363 [ENHANCEMENT] Ruler: Include the tenant ID on the notifier logs. #3372 [ENHANCEMENT] Blocks storage Compactor: Added -compactor.enabled-tenants and -compactor.disabled-tenants to explicitly enable or disable compaction of specific tenants. #3385 [ENHANCEMENT] Blocks storage ingester: Creating checkpoint only once even when there are multiple Head compactions in a single Compact() call. #3373 [BUGFIX] Blocks storage ingester: Read repair memory-mapped chunks file which can end up being empty on abrupt shutdowns combined with faulty disks. #3373 [BUGFIX] Blocks storage ingester: Close TSDB resources on failed startup preventing ingester OOMing. #3373 [BUGFIX] No-longer-needed ingester operations for queries triggered by queriers and rulers are now canceled. #3178 [BUGFIX] Ruler: directories in the configured rules-path will be removed on startup and shutdown in order to ensure they don’t persist between runs. #3195 [BUGFIX] Handle hash-collisions in the query path. #3192 [BUGFIX] Check for postgres rows errors. #3197 [BUGFIX] Ruler Experimental API: Don’t allow rule groups without names or empty rule groups. #3210 [BUGFIX] Experimental Alertmanager API: Do not allow empty Alertmanager configurations or bad template filenames to be submitted through the configuration API. #3185 [BUGFIX] Reduce failures to update heartbeat when using Consul. #3259 [BUGFIX] When using ruler sharding, moving all user rule groups from ruler to a different one and then back could end up with some user groups not being evaluated at all. #3235 [BUGFIX] Fixed shuffle sharding consistency when zone-awareness is enabled and the shard size is increased or instances in a new zone are added. #3299 [BUGFIX] Use a valid grpc header when logging IP addresses. #3307 [BUGFIX] Fixed the metric cortex_prometheus_rule_group_duration_seconds in the Ruler, it wouldn’t report any values. #3310 [BUGFIX] Fixed gRPC connections leaking in rulers when rulers sharding is enabled and APIs called. #3314 [BUGFIX] Fixed shuffle sharding consistency when zone-awareness is enabled and the shard size is increased or instances in a new zone are added. #3299 [BUGFIX] Fixed Gossip memberlist members joining when addresses are configured using DNS-based service discovery. #3360 [BUGFIX] Ingester: fail to start an ingester running the blocks storage, if unable to load any existing TSDB at startup. #3354 [BUGFIX] Blocks storage: Avoid deletion of blocks in the ingester which are not shipped to the storage yet. #3346 [BUGFIX] Fix common prefixes returned by List method of S3 client. #3358 [BUGFIX] Honor configured timeout in Azure and GCS object clients. #3285 [BUGFIX] Blocks storage: Avoid creating blocks larger than configured block range period on forced compaction and when TSDB is idle. #3344 [BUGFIX] Shuffle sharding: fixed max global series per user/metric limit when shuffle sharding and -distributor.shard-by-all-labels=true are both enabled in distributor. When using these global limits you should now set -distributor.sharding-strategy and -distributor.zone-awareness-enabled to ingesters too. #3369 [BUGFIX] Slow query logging: when using downstream server request parameters were not logged. #3276 [BUGFIX] Fixed tenant detection in the ruler and alertmanager API when running without auth. #3343 Blocksconvert [ENHANCEMENT] Blocksconvert – Builder: download plan file locally before processing it. #3209 [ENHANCEMENT] Blocksconvert – Cleaner: added new tool for deleting chunks data. #3283 [ENHANCEMENT] Blocksconvert – Scanner: support for scanning specific date-range only. #3222 [ENHANCEMENT] Blocksconvert – Scanner: metrics for tracking progress. #3222 [ENHANCEMENT] Blocksconvert – Builder: retry block upload before giving up. #3245 [ENHANCEMENT] Blocksconvert – Scanner: upload plans concurrently. #3340 [BUGFIX] Blocksconvert: fix chunks ordering in the block. Chunks in different order than series work just fine in TSDB blocks at the moment, but it’s not consistent with what Prometheus does and future Prometheus and Cortex optimizations may rely on this ordering. #3371 1.4.0 / 2020-10-02 [CHANGE] TLS configuration for gRPC, HTTP and etcd clients is now marked as experimental. These features are not yet fully baked, and we expect possible small breaking changes in Cortex 1.5. #3198 [CHANGE] Cassandra backend support is now GA (stable). #3180 [CHANGE] Blocks storage is now GA (stable). The -experimental prefix has been removed from all CLI flags related to the blocks storage (no YAML config changes). #3180 #3201 -experimental.blocks-storage.* flags renamed to -blocks-storage.* -experimental.store-gateway.* flags renamed to -store-gateway.* -experimental.querier.store-gateway-client.* flags renamed to -querier.store-gateway-client.* -experimental.querier.store-gateway-addresses flag renamed to -querier.store-gateway-addresses -store-gateway.replication-factor flag renamed to -store-gateway.sharding-ring.replication-factor -store-gateway.tokens-file-path flag renamed to store-gateway.sharding-ring.tokens-file-path [CHANGE] Ingester: Removed deprecated untyped record from chunks WAL. Only if you are running v1.0 or below, it is recommended to first upgrade to v1.1/v1.2/v1.3 and run it for a day before upgrading to v1.4 to avoid data loss. #3115 [CHANGE] Distributor API endpoints are no longer served unless target is set to distributor or all. #3112 [CHANGE] Increase the default Cassandra client replication factor to 3. #3007 [CHANGE] Blocks storage: removed the support to transfer blocks between ingesters on shutdown. When running the Cortex blocks storage, ingesters are expected to run with a persistent disk. The following metrics have been removed: #2996 cortex_ingester_sent_files cortex_ingester_received_files cortex_ingester_received_bytes_total cortex_ingester_sent_bytes_total [CHANGE] The buckets for the cortex_chunk_store_index_lookups_per_query metric have been changed to 1, 2, 4, 8, 16. #3021 [CHANGE] Blocks storage: the operation label value getrange has changed into get_range for the metrics thanos_store_bucket_cache_operation_requests_total and thanos_store_bucket_cache_operation_hits_total. #3000 [CHANGE] Experimental Delete Series: /api/v1/admin/tsdb/delete_series and /api/v1/admin/tsdb/cancel_delete_request purger APIs to return status code 204 instead of 200 for success. #2946 [CHANGE] Histogram cortex_memcache_request_duration_seconds method label value changes from Memcached.Get to Memcached.GetBatched for batched lookups, and is not reported for non-batched lookups (label value Memcached.GetMulti remains, and had exactly the same value as Get in nonbatched lookups). The same change applies to tracing spans. #3046 [CHANGE] TLS server validation is now enabled by default, a new parameter tls_insecure_skip_verify can be set to true to skip validation optionally. #3030 [CHANGE] cortex_ruler_config_update_failures_total has been removed in favor of cortex_ruler_config_last_reload_successful. #3056 [CHANGE] ruler.evaluation_delay_duration field in YAML config has been moved and renamed to limits.ruler_evaluation_delay_duration. #3098 [CHANGE] Removed obsolete results_cache.max_freshness from YAML config (deprecated since Cortex 1.2). #3145 [CHANGE] Removed obsolete -promql.lookback-delta option (deprecated since Cortex 1.2, replaced with -querier.lookback-delta). #3144 [CHANGE] Cache: added support for Redis Cluster and Redis Sentinel. #2961 The following changes have been made in Redis configuration: -redis.master_name added -redis.db added -redis.max-active-conns changed to -redis.pool-size -redis.max-conn-lifetime changed to -redis.max-connection-age -redis.max-idle-conns removed -redis.wait-on-pool-exhaustion removed [CHANGE] TLS configuration for gRPC, HTTP and etcd clients is now marked as experimental. These features are not yet fully baked, and we expect possible small breaking changes in Cortex 1.5. #3198 [CHANGE] Fixed store-gateway CLI flags inconsistencies. #3201 -store-gateway.replication-factor flag renamed to -store-gateway.sharding-ring.replication-factor -store-gateway.tokens-file-path flag renamed to store-gateway.sharding-ring.tokens-file-path [FEATURE] Logging of the source IP passed along by a reverse proxy is now supported by setting the -server.log-source-ips-enabled. For non standard headers the settings -server.log-source-ips-header and -server.log-source-ips-regex can be used. #2985 [FEATURE] Blocks storage: added shuffle sharding support to store-gateway blocks sharding. Added the following additional metrics to store-gateway: #3069 cortex_bucket_stores_tenants_discovered cortex_bucket_stores_tenants_synced [FEATURE] Experimental blocksconvert: introduce an experimental tool blocksconvert to migrate long-term storage chunks to blocks. #3092 #3122 #3127 #3162 [ENHANCEMENT] Improve the Alertmanager logging when serving requests from its API / UI. #3397 [ENHANCEMENT] Add support for azure storage in China, German and US Government environments. #2988 [ENHANCEMENT] Query-tee: added a small tolerance to floating point sample values comparison. #2994 [ENHANCEMENT] Query-tee: add support for doing a passthrough of requests to preferred backend for unregistered routes #3018 [ENHANCEMENT] Expose storage.aws.dynamodb.backoff_config configuration file field. #3026 [ENHANCEMENT] Added cortex_request_message_bytes and cortex_response_message_bytes histograms to track received and sent gRPC message and HTTP request/response sizes. Added cortex_inflight_requests gauge to track number of inflight gRPC and HTTP requests. #3064 [ENHANCEMENT] Publish ruler’s ring metrics. #3074 [ENHANCEMENT] Add config validation to the experimental Alertmanager API. Invalid configs are no longer accepted. #3053 [ENHANCEMENT] Add “integration” as a label for cortex_alertmanager_notifications_total and cortex_alertmanager_notifications_failed_total metrics. #3056 [ENHANCEMENT] Add cortex_ruler_config_last_reload_successful and cortex_ruler_config_last_reload_successful_seconds to check status of users rule manager. #3056 [ENHANCEMENT] The configuration validation now fails if an empty YAML node has been set for a root YAML config property. #3080 [ENHANCEMENT] Memcached dial() calls now have a circuit-breaker to avoid hammering a broken cache. #3051, #3189 [ENHANCEMENT] -ruler.evaluation-delay-duration is now overridable as a per-tenant limit, ruler_evaluation_delay_duration. #3098 [ENHANCEMENT] Add TLS support to etcd client. #3102 [ENHANCEMENT] When a tenant accesses the Alertmanager UI or its API, if we have valid -alertmanager.configs.fallback we’ll use that to start the manager and avoid failing the request. #3073 [ENHANCEMENT] Add DELETE api/v1/rules/{namespace} to the Ruler. It allows all the rule groups of a namespace to be deleted. #3120 [ENHANCEMENT] Experimental Delete Series: Retry processing of Delete requests during failures. #2926 [ENHANCEMENT] Improve performance of QueryStream() in ingesters. #3177 [ENHANCEMENT] Modules included in “All” target are now visible in output of -modules CLI flag. #3155 [ENHANCEMENT] Added /debug/fgprof endpoint to debug running Cortex process using fgprof. This adds up to the existing /debug/... endpoints. #3131 [ENHANCEMENT] Blocks storage: optimised /api/v1/series for blocks storage. (#2976) [BUGFIX] Ruler: when loading rules from “local” storage, check for directory after resolving symlink. #3137 [BUGFIX] Query-frontend: Fixed rounding for incoming query timestamps, to be 100% Prometheus compatible. #2990 [BUGFIX] Querier: Merge results from chunks and blocks ingesters when using streaming of results. #3013 [BUGFIX] Querier: query /series from ingesters regardless the -querier.query-ingesters-within setting. #3035 [BUGFIX] Blocks storage: Ingester is less likely to hit gRPC message size limit when streaming data to queriers. #3015 [BUGFIX] Blocks storage: fixed memberlist support for the store-gateways and compactors ring used when blocks sharding is enabled. #3058 #3095 [BUGFIX] Fix configuration for TLS server validation, TLS skip verify was hardcoded to true for all TLS configurations and prevented validation of server certificates. #3030 [BUGFIX] Fixes the Alertmanager panicking when no -alertmanager.web.external-url is provided. #3017 [BUGFIX] Fixes the registration of the Alertmanager API metrics cortex_alertmanager_alerts_received_total and cortex_alertmanager_alerts_invalid_total. #3065 [BUGFIX] Fixes flag needs an argument: -config.expand-env error. #3087 [BUGFIX] An index optimisation actually slows things down when using caching. Moved it to the right location. #2973 [BUGFIX] Ingester: If push request contained both valid and invalid samples, valid samples were ingested but not stored to WAL of the chunks storage. This has been fixed. #3067 [BUGFIX] Cassandra: fixed consistency setting in the CQL session when creating the keyspace. #3105 [BUGFIX] Ruler: Config API would return both the record and alert in YAML response keys even when one of them must be empty. #3120 [BUGFIX] Index page now uses configured HTTP path prefix when creating links. #3126 [BUGFIX] Purger: fixed deadlock when reloading of tombstones failed. #3182 [BUGFIX] Fixed panic in flusher job, when error writing chunks to the store would cause “idle” chunks to be flushed, which triggered panic. #3140 [BUGFIX] Index page no longer shows links that are not valid for running Cortex instance. #3133 [BUGFIX] Configs: prevent validation of templates to fail when using template functions. #3157 [BUGFIX] Configuring the S3 URL with an @ but without username and password doesn’t enable the AWS static credentials anymore. #3170 [BUGFIX] Limit errors on ranged queries (api/v1/query_range) no longer return a status code 500 but 422 instead. #3167 [BUGFIX] Handle hash-collisions in the query path. Before this fix, Cortex could occasionally mix up two different series in a query, leading to invalid results, when -querier.ingester-streaming was used. #3192 1.3.0 / 2020-08-21 [CHANGE] Replace the metric cortex_alertmanager_configs with cortex_alertmanager_config_invalid exposed by Alertmanager. #2960 [CHANGE] Experimental Delete Series: Change target flag for purger from data-purger to purger. #2777 [CHANGE] Experimental blocks storage: The max concurrent queries against the long-term storage, configured via -experimental.blocks-storage.bucket-store.max-concurrent, is now a limit shared across all tenants and not a per-tenant limit anymore. The default value has changed from 20 to 100 and the following new metrics have been added: #2797 cortex_bucket_stores_gate_queries_concurrent_max cortex_bucket_stores_gate_queries_in_flight cortex_bucket_stores_gate_duration_seconds [CHANGE] Metric cortex_ingester_flush_reasons has been renamed to cortex_ingester_flushing_enqueued_series_total, and new metric cortex_ingester_flushing_dequeued_series_total with outcome label (superset of reason) has been added. #2802 #2818 #2998 [CHANGE] Experimental Delete Series: Metric cortex_purger_oldest_pending_delete_request_age_seconds would track age of delete requests since they are over their cancellation period instead of their creation time. #2806 [CHANGE] Experimental blocks storage: the store-gateway service is required in a Cortex cluster running with the experimental blocks storage. Removed the -experimental.tsdb.store-gateway-enabled CLI flag and store_gateway_enabled YAML config option. The store-gateway is now always enabled when the storage engine is blocks. #2822 [CHANGE] Experimental blocks storage: removed support for -experimental.blocks-storage.bucket-store.max-sample-count flag because the implementation was flawed. To limit the number of samples/chunks processed by a single query you can set -store.query-chunk-limit, which is now supported by the blocks storage too. #2852 [CHANGE] Ingester: Chunks flushed via /flush stay in memory until retention period is reached. This affects cortex_ingester_memory_chunks metric. #2778 [CHANGE] Querier: the error message returned when the query time range exceeds -store.max-query-length has changed from invalid query, length \u003e limit (X \u003e Y) to the query time range exceeds the limit (query length: X, limit: Y). #2826 [CHANGE] Add component label to metrics exposed by chunk, delete and index store clients. #2774 [CHANGE] Querier: when -querier.query-ingesters-within is configured, the time range of the query sent to ingesters is now manipulated to ensure the query start time is not older than ’now - query-ingesters-within’. #2904 [CHANGE] KV: The role label which was a label of multi KV store client only has been added to metrics of every KV store client. If KV store client is not multi, then the value of role label is primary. #2837 [CHANGE] Added the engine label to the metrics exposed by the Prometheus query engine, to distinguish between ruler and querier metrics. #2854 [CHANGE] Added ruler to the single binary when started with -target=all (default). #2854 [CHANGE] Experimental blocks storage: compact head when opening TSDB. This should only affect ingester startup after it was unable to compact head in previous run. #2870 [CHANGE] Metric cortex_overrides_last_reload_successful has been renamed to cortex_runtime_config_last_reload_successful. #2874 [CHANGE] HipChat support has been removed from the alertmanager (because removed from the Prometheus upstream too). #2902 [CHANGE] Add constant label name to metric cortex_cache_request_duration_seconds. #2903 [CHANGE] Add user label to metric cortex_query_frontend_queue_length. #2939 [CHANGE] Experimental blocks storage: cleaned up the config and renamed “TSDB” to “blocks storage”. #2937 The storage engine setting value has been changed from tsdb to blocks; this affects -store.engine CLI flag and its respective YAML option. The root level YAML config has changed from tsdb to blocks_storage The prefix of all CLI flags has changed from -experimental.tsdb. to -experimental.blocks-storage. The following settings have been grouped under tsdb property in the YAML config and their CLI flags changed: -experimental.tsdb.dir changed to -experimental.blocks-storage.tsdb.dir -experimental.tsdb.block-ranges-period changed to -experimental.blocks-storage.tsdb.block-ranges-period -experimental.tsdb.retention-period changed to -experimental.blocks-storage.tsdb.retention-period -experimental.tsdb.ship-interval changed to -experimental.blocks-storage.tsdb.ship-interval -experimental.tsdb.ship-concurrency changed to -experimental.blocks-storage.tsdb.ship-concurrency -experimental.tsdb.max-tsdb-opening-concurrency-on-startup changed to -experimental.blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup -experimental.tsdb.head-compaction-interval changed to -experimental.blocks-storage.tsdb.head-compaction-interval -experimental.tsdb.head-compaction-concurrency changed to -experimental.blocks-storage.tsdb.head-compaction-concurrency -experimental.tsdb.head-compaction-idle-timeout changed to -experimental.blocks-storage.tsdb.head-compaction-idle-timeout -experimental.tsdb.stripe-size changed to -experimental.blocks-storage.tsdb.stripe-size -experimental.tsdb.wal-compression-enabled changed to -experimental.blocks-storage.tsdb.wal-compression-enabled -experimental.tsdb.flush-blocks-on-shutdown changed to -experimental.blocks-storage.tsdb.flush-blocks-on-shutdown [CHANGE] Flags -bigtable.grpc-use-gzip-compression, -ingester.client.grpc-use-gzip-compression, -querier.frontend-client.grpc-use-gzip-compression are now deprecated. #2940 [CHANGE] Limit errors reported by ingester during query-time now return HTTP status code 422. #2941 [FEATURE] Introduced ruler.for-outage-tolerance, Max time to tolerate outage for restoring “for” state of alert. #2783 [FEATURE] Introduced ruler.for-grace-period, Minimum duration between alert and restored “for” state. This is maintained only for alerts with configured “for” time greater than grace period. #2783 [FEATURE] Introduced ruler.resend-delay, Minimum amount of time to wait before resending an alert to Alertmanager. #2783 [FEATURE] Ruler: added local filesystem support to store rules (read-only). #2854 [ENHANCEMENT] Upgraded Docker base images to alpine:3.12. #2862 [ENHANCEMENT] Experimental: Querier can now optionally query secondary store. This is specified by using -querier.second-store-engine option, with values chunks or blocks. Standard configuration options for this store are used. Additionally, this querying can be configured to happen only for queries that need data older than -querier.use-second-store-before-time. Default value of zero will always query secondary store. #2747 [ENHANCEMENT] Query-tee: increased the cortex_querytee_request_duration_seconds metric buckets granularity. #2799 [ENHANCEMENT] Query-tee: fail to start if the configured -backend.preferred is unknown. #2799 [ENHANCEMENT] Ruler: Added the following metrics: #2786 cortex_prometheus_notifications_latency_seconds cortex_prometheus_notifications_errors_total cortex_prometheus_notifications_sent_total cortex_prometheus_notifications_dropped_total cortex_prometheus_notifications_queue_length cortex_prometheus_notifications_queue_capacity cortex_prometheus_notifications_alertmanagers_discovered [ENHANCEMENT] The behavior of the /ready was changed for the query frontend to indicate when it was ready to accept queries. This is intended for use by a read path load balancer that would want to wait for the frontend to have attached queriers before including it in the backend. #2733 [ENHANCEMENT] Experimental Delete Series: Add support for deletion of chunks for remaining stores. #2801 [ENHANCEMENT] Add -modules command line flag to list possible values for -target. Also, log warning if given target is internal component. #2752 [ENHANCEMENT] Added -ingester.flush-on-shutdown-with-wal-enabled option to enable chunks flushing even when WAL is enabled. #2780 [ENHANCEMENT] Query-tee: Support for custom API prefix by using -server.path-prefix option. #2814 [ENHANCEMENT] Query-tee: Forward X-Scope-OrgId header to backend, if present in the request. #2815 [ENHANCEMENT] Experimental blocks storage: Added -experimental.blocks-storage.tsdb.head-compaction-idle-timeout option to force compaction of data in memory into a block. #2803 [ENHANCEMENT] Experimental blocks storage: Added support for flushing blocks via /flush, /shutdown (previously these only worked for chunks storage) and by using -experimental.blocks-storage.tsdb.flush-blocks-on-shutdown option. #2794 [ENHANCEMENT] Experimental blocks storage: Added support to enforce max query time range length via -store.max-query-length. #2826 [ENHANCEMENT] Experimental blocks storage: Added support to limit the max number of chunks that can be fetched from the long-term storage while executing a query. The limit is enforced both in the querier and store-gateway, and is configurable via -store.query-chunk-limit. #2852 #2922 [ENHANCEMENT] Ingester: Added new metric cortex_ingester_flush_series_in_progress that reports number of ongoing flush-series operations. Useful when calling /flush handler: if cortex_ingester_flush_queue_length + cortex_ingester_flush_series_in_progress is 0, all flushes are finished. #2778 [ENHANCEMENT] Memberlist members can join cluster via SRV records. #2788 [ENHANCEMENT] Added configuration options for chunks s3 client. #2831 s3.endpoint s3.region s3.access-key-id s3.secret-access-key s3.insecure s3.sse-encryption s3.http.idle-conn-timeout s3.http.response-header-timeout s3.http.insecure-skip-verify [ENHANCEMENT] Prometheus upgraded. #2798 #2849 #2867 #2902 #2918 Optimized labels regex matchers for patterns containing literals (eg. foo.*, .*foo, .*foo.*) [ENHANCEMENT] Add metric cortex_ruler_config_update_failures_total to Ruler to track failures of loading rules files. #2857 [ENHANCEMENT] Experimental Alertmanager: Alertmanager configuration persisted to object storage using an experimental API that accepts and returns YAML-based Alertmanager configuration. #2768 [ENHANCEMENT] Ruler: -ruler.alertmanager-url now supports multiple URLs. Each URL is treated as a separate Alertmanager group. Support for multiple Alertmanagers in a group can be achieved by using DNS service discovery. #2851 [ENHANCEMENT] Experimental blocks storage: Cortex Flusher now works with blocks engine. Flusher needs to be provided with blocks-engine configuration, existing Flusher flags are not used (they are only relevant for chunks engine). Note that flush errors are only reported via log. #2877 [ENHANCEMENT] Flusher: Added -flusher.exit-after-flush option (defaults to true) to control whether Cortex should stop completely after Flusher has finished its work. #2877 [ENHANCEMENT] Added metrics cortex_config_hash and cortex_runtime_config_hash to expose hash of the currently active config file. #2874 [ENHANCEMENT] Logger: added JSON logging support, configured via the -log.format=json CLI flag or its respective YAML config option. #2386 [ENHANCEMENT] Added new flags -bigtable.grpc-compression, -ingester.client.grpc-compression, -querier.frontend-client.grpc-compression to configure compression used by gRPC. Valid values are gzip, snappy, or empty string (no compression, default). #2940 [ENHANCEMENT] Clarify limitations of the /api/v1/series, /api/v1/labels and /api/v1/label/{name}/values endpoints. #2953 [ENHANCEMENT] Ingester: added Dropped outcome to metric cortex_ingester_flushing_dequeued_series_total. #2998 [BUGFIX] Fixed a bug with api/v1/query_range where no responses would return null values for result and empty values for resultType. #2962 [BUGFIX] Fixed a bug in the index intersect code causing storage to return more chunks/series than required. #2796 [BUGFIX] Fixed the number of reported keys in the background cache queue. #2764 [BUGFIX] Fix race in processing of headers in sharded queries. #2762 [BUGFIX] Query Frontend: Do not re-split sharded requests around ingester boundaries. #2766 [BUGFIX] Experimental Delete Series: Fixed a problem with cache generation numbers prefixed to cache keys. #2800 [BUGFIX] Ingester: Flushing chunks via /flush endpoint could previously lead to panic, if chunks were already flushed before and then removed from memory during the flush caused by /flush handler. Immediate flush now doesn’t cause chunks to be flushed again. Samples received during flush triggered via /flush handler are no longer discarded. #2778 [BUGFIX] Prometheus upgraded. #2849 Fixed unknown symbol error during head compaction [BUGFIX] Fix panic when using cassandra as store for both index and delete requests. #2774 [BUGFIX] Experimental Delete Series: Fixed a data race in Purger. #2817 [BUGFIX] KV: Fixed a bug that triggered a panic due to metrics being registered with the same name but different labels when using a multi configured KV client. #2837 [BUGFIX] Query-frontend: Fix passing HTTP Host header if -frontend.downstream-url is configured. #2880 [BUGFIX] Ingester: Improve time-series distribution when -experimental.distributor.user-subring-size is enabled. #2887 [BUGFIX] Set content type to application/x-protobuf for remote_read responses. #2915 [BUGFIX] Fixed ruler and store-gateway instance registration in the ring (when sharding is enabled) when a new instance replaces abruptly terminated one, and the only difference between the two instances is the address. #2954 [BUGFIX] Fixed Missing chunks and index config causing silent failure Absence of chunks and index from schema config is not validated. #2732 [BUGFIX] Fix panic caused by KVs from boltdb being used beyond their life. #2971 [BUGFIX] Experimental blocks storage: /api/v1/series, /api/v1/labels and /api/v1/label/{name}/values only query the TSDB head regardless of the configured -experimental.blocks-storage.tsdb.retention-period. #2974 [BUGFIX] Ingester: Avoid indefinite checkpointing in case of surge in number of series. #2955 [BUGFIX] Querier: query /series from ingesters regardless the -querier.query-ingesters-within setting. #3035 [BUGFIX] Ruler: fixed an unintentional breaking change introduced in the ruler’s alertmanager_url YAML config option, which changed the value from a string to a list of strings. #2989 1.2.0 / 2020-07-01 [CHANGE] Metric cortex_kv_request_duration_seconds now includes name label to denote which client is being used as well as the backend label to denote the KV backend implementation in use. #2648 [CHANGE] Experimental Ruler: Rule groups persisted to object storage using the experimental API have an updated object key encoding to better handle special characters. Rule groups previously-stored using object storage must be renamed to the new format. #2646 [CHANGE] Query Frontend now uses Round Robin to choose a tenant queue to service next. #2553 [CHANGE] -promql.lookback-delta is now deprecated and has been replaced by -querier.lookback-delta along with lookback_delta entry under querier in the config file. -promql.lookback-delta will be removed in v1.4.0. #2604 [CHANGE] Experimental TSDB: removed -experimental.tsdb.bucket-store.binary-index-header-enabled flag. Now the binary index-header is always enabled. [CHANGE] Experimental TSDB: Renamed index-cache metrics to use original metric names from Thanos, as Cortex is not aggregating them in any way: #2627 cortex_\u003cservice\u003e_blocks_index_cache_items_evicted_total =\u003e thanos_store_index_cache_items_evicted_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_items_added_total =\u003e thanos_store_index_cache_items_added_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_requests_total =\u003e thanos_store_index_cache_requests_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_items_overflowed_total =\u003e thanos_store_index_cache_items_overflowed_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_hits_total =\u003e thanos_store_index_cache_hits_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_items =\u003e thanos_store_index_cache_items{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_items_size_bytes =\u003e thanos_store_index_cache_items_size_bytes{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_total_size_bytes =\u003e thanos_store_index_cache_total_size_bytes{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_memcached_operations_total =\u003e thanos_memcached_operations_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_memcached_operation_failures_total =\u003e thanos_memcached_operation_failures_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_memcached_operation_duration_seconds =\u003e thanos_memcached_operation_duration_seconds{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_memcached_operation_skipped_total =\u003e thanos_memcached_operation_skipped_total{name=\"index-cache\"} [CHANGE] Experimental TSDB: Renamed metrics in bucket stores: #2627 cortex_\u003cservice\u003e_blocks_meta_syncs_total =\u003e cortex_blocks_meta_syncs_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_blocks_meta_sync_failures_total =\u003e cortex_blocks_meta_sync_failures_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_blocks_meta_sync_duration_seconds =\u003e cortex_blocks_meta_sync_duration_seconds{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_blocks_meta_sync_consistency_delay_seconds =\u003e cortex_blocks_meta_sync_consistency_delay_seconds{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_blocks_meta_synced =\u003e cortex_blocks_meta_synced{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_block_loads_total =\u003e cortex_bucket_store_block_loads_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_block_load_failures_total =\u003e cortex_bucket_store_block_load_failures_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_block_drops_total =\u003e cortex_bucket_store_block_drops_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_block_drop_failures_total =\u003e cortex_bucket_store_block_drop_failures_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_blocks_loaded =\u003e cortex_bucket_store_blocks_loaded{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_data_touched =\u003e cortex_bucket_store_series_data_touched{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_data_fetched =\u003e cortex_bucket_store_series_data_fetched{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_data_size_touched_bytes =\u003e cortex_bucket_store_series_data_size_touched_bytes{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_data_size_fetched_bytes =\u003e cortex_bucket_store_series_data_size_fetched_bytes{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_blocks_queried =\u003e cortex_bucket_store_series_blocks_queried{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_get_all_duration_seconds =\u003e cortex_bucket_store_series_get_all_duration_seconds{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_merge_duration_seconds =\u003e cortex_bucket_store_series_merge_duration_seconds{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_refetches_total =\u003e cortex_bucket_store_series_refetches_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_result_series =\u003e cortex_bucket_store_series_result_series{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_cached_postings_compressions_total =\u003e cortex_bucket_store_cached_postings_compressions_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_cached_postings_compression_errors_total =\u003e cortex_bucket_store_cached_postings_compression_errors_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_cached_postings_compression_time_seconds =\u003e cortex_bucket_store_cached_postings_compression_time_seconds{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_cached_postings_original_size_bytes_total =\u003e cortex_bucket_store_cached_postings_original_size_bytes_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_cached_postings_compressed_size_bytes_total =\u003e cortex_bucket_store_cached_postings_compressed_size_bytes_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_blocks_sync_seconds =\u003e cortex_bucket_stores_blocks_sync_seconds{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_blocks_last_successful_sync_timestamp_seconds =\u003e cortex_bucket_stores_blocks_last_successful_sync_timestamp_seconds{component=\"\u003cservice\u003e\"} [CHANGE] Available command-line flags are printed to stdout, and only when requested via -help. Using invalid flag no longer causes printing of all available flags. #2691 [CHANGE] Experimental Memberlist ring: randomize gossip node names to avoid conflicts when running multiple clients on the same host, or reusing host names (eg. pods in statefulset). Node name randomization can be disabled by using -memberlist.randomize-node-name=false. #2715 [CHANGE] Memberlist KV client is no longer considered experimental. #2725 [CHANGE] Experimental Delete Series: Make delete request cancellation duration configurable. #2760 [CHANGE] Removed -store.fullsize-chunks option which was undocumented and unused (it broke ingester hand-overs). #2656 [CHANGE] Query with no metric name that has previously resulted in HTTP status code 500 now returns status code 422 instead. #2571 [FEATURE] TLS config options added for GRPC clients in Querier (Query-frontend client \u0026 Ingester client), Ruler, Store Gateway, as well as HTTP client in Config store client. #2502 [FEATURE] The flag -frontend.max-cache-freshness is now supported within the limits overrides, to specify per-tenant max cache freshness values. The corresponding YAML config parameter has been changed from results_cache.max_freshness to limits_config.max_cache_freshness. The legacy YAML config parameter (results_cache.max_freshness) will continue to be supported till Cortex release v1.4.0. #2609 [FEATURE] Experimental gRPC Store: Added support to 3rd parties index and chunk stores using gRPC client/server plugin mechanism. #2220 [FEATURE] Add -cassandra.table-options flag to customize table options of Cassandra when creating the index or chunk table. #2575 [ENHANCEMENT] Propagate GOPROXY value when building build-image. This is to help the builders building the code in a Network where default Go proxy is not accessible (e.g. when behind some corporate VPN). #2741 [ENHANCEMENT] Querier: Added metric cortex_querier_request_duration_seconds for all requests to the querier. #2708 [ENHANCEMENT] Cortex is now built with Go 1.14. #2480 #2749 #2753 [ENHANCEMENT] Experimental TSDB: added the following metrics to the ingester: #2580 #2583 #2589 #2654 cortex_ingester_tsdb_appender_add_duration_seconds cortex_ingester_tsdb_appender_commit_duration_seconds cortex_ingester_tsdb_refcache_purge_duration_seconds cortex_ingester_tsdb_compactions_total cortex_ingester_tsdb_compaction_duration_seconds cortex_ingester_tsdb_wal_fsync_duration_seconds cortex_ingester_tsdb_wal_page_flushes_total cortex_ingester_tsdb_wal_completed_pages_total cortex_ingester_tsdb_wal_truncations_failed_total cortex_ingester_tsdb_wal_truncations_total cortex_ingester_tsdb_wal_writes_failed_total cortex_ingester_tsdb_checkpoint_deletions_failed_total cortex_ingester_tsdb_checkpoint_deletions_total cortex_ingester_tsdb_checkpoint_creations_failed_total cortex_ingester_tsdb_checkpoint_creations_total cortex_ingester_tsdb_wal_truncate_duration_seconds cortex_ingester_tsdb_head_active_appenders cortex_ingester_tsdb_head_series_not_found_total cortex_ingester_tsdb_head_chunks cortex_ingester_tsdb_mmap_chunk_corruptions_total cortex_ingester_tsdb_head_chunks_created_total cortex_ingester_tsdb_head_chunks_removed_total [ENHANCEMENT] Experimental TSDB: added metrics useful to alert on critical conditions of the blocks storage: #2573 cortex_compactor_last_successful_run_timestamp_seconds cortex_querier_blocks_last_successful_sync_timestamp_seconds (when store-gateway is disabled) cortex_querier_blocks_last_successful_scan_timestamp_seconds (when store-gateway is enabled) cortex_storegateway_blocks_last_successful_sync_timestamp_seconds [ENHANCEMENT] Experimental TSDB: added the flag -experimental.tsdb.wal-compression-enabled to allow to enable TSDB WAL compression. #2585 [ENHANCEMENT] Experimental TSDB: Querier and store-gateway components can now use so-called “caching bucket”, which can currently cache fetched chunks into shared memcached server. #2572 [ENHANCEMENT] Ruler: Automatically remove unhealthy rulers from the ring. #2587 [ENHANCEMENT] Query-tee: added support to /metadata, /alerts, and /rules endpoints #2600 [ENHANCEMENT] Query-tee: added support to query results comparison between two different backends. The comparison is disabled by default and can be enabled via -proxy.compare-responses=true. #2611 [ENHANCEMENT] Query-tee: improved the query-tee to not wait all backend responses before sending back the response to the client. The query-tee now sends back to the client first successful response, while honoring the -backend.preferred option. #2702 [ENHANCEMENT] Thanos and Prometheus upgraded. #2602 #2604 #2634 #2659 #2686 #2756 TSDB now holds less WAL files after Head Truncation. TSDB now does memory-mapping of Head chunks and reduces memory usage. [ENHANCEMENT] Experimental TSDB: decoupled blocks deletion from blocks compaction in the compactor, so that blocks deletion is not blocked by a busy compactor. The following metrics have been added: #2623 cortex_compactor_block_cleanup_started_total cortex_compactor_block_cleanup_completed_total cortex_compactor_block_cleanup_failed_total cortex_compactor_block_cleanup_last_successful_run_timestamp_seconds [ENHANCEMENT] Experimental TSDB: Use shared cache for metadata. This is especially useful when running multiple querier and store-gateway components to reduce number of object store API calls. #2626 #2640 [ENHANCEMENT] Experimental TSDB: when -querier.query-store-after is configured and running the experimental blocks storage, the time range of the query sent to the store is now manipulated to ensure the query end time is not more recent than ’now - query-store-after’. #2642 [ENHANCEMENT] Experimental TSDB: small performance improvement in concurrent usage of RefCache, used during samples ingestion. #2651 [ENHANCEMENT] The following endpoints now respond appropriately to an Accept header with the value application/json #2673 /distributor/all_user_stats /distributor/ha_tracker /ingester/ring /store-gateway/ring /compactor/ring /ruler/ring /services [ENHANCEMENT] Experimental Cassandra backend: Add -cassandra.num-connections to allow increasing the number of TCP connections to each Cassandra server. #2666 [ENHANCEMENT] Experimental Cassandra backend: Use separate Cassandra clients and connections for reads and writes. #2666 [ENHANCEMENT] Experimental Cassandra backend: Add -cassandra.reconnect-interval to allow specifying the reconnect interval to a Cassandra server that has been marked DOWN by the gocql driver. Also change the default value of the reconnect interval from 60s to 1s. #2687 [ENHANCEMENT] Experimental Cassandra backend: Add option -cassandra.convict-hosts-on-failure=false to not convict host of being down when a request fails. #2684 [ENHANCEMENT] Experimental TSDB: Applied a jitter to the period bucket scans in order to better distribute bucket operations over the time and increase the probability of hitting the shared cache (if configured). #2693 [ENHANCEMENT] Experimental TSDB: Series limit per user and per metric now work in TSDB blocks. #2676 [ENHANCEMENT] Experimental Memberlist: Added ability to periodically rejoin the memberlist cluster. #2724 [ENHANCEMENT] Experimental Delete Series: Added the following metrics for monitoring processing of delete requests: #2730 cortex_purger_load_pending_requests_attempts_total: Number of attempts that were made to load pending requests with status. cortex_purger_oldest_pending_delete_request_age_seconds: Age of oldest pending delete request in seconds. cortex_purger_pending_delete_requests_count: Count of requests which are in process or are ready to be processed. [ENHANCEMENT] Experimental TSDB: Improved compactor to hard-delete also partial blocks with an deletion mark (even if the deletion mark threshold has not been reached). #2751 [ENHANCEMENT] Experimental TSDB: Introduced a consistency check done by the querier to ensure all expected blocks have been queried via the store-gateway. If a block is missing on a store-gateway, the querier retries fetching series from missing blocks up to 3 times. If the consistency check fails once all retries have been exhausted, the query execution fails. The following metrics have been added: #2593 #2630 #2689 #2695 cortex_querier_blocks_consistency_checks_total cortex_querier_blocks_consistency_checks_failed_total cortex_querier_storegateway_refetches_per_query [ENHANCEMENT] Delete requests can now be canceled #2555 [ENHANCEMENT] Table manager can now provision tables for delete store #2546 [BUGFIX] Ruler: Ensure temporary rule files with special characters are properly mapped and cleaned up. #2506 [BUGFIX] Fixes #2411, Ensure requests are properly routed to the prometheus api embedded in the query if -server.path-prefix is set. #2372 [BUGFIX] Experimental TSDB: fixed chunk data corruption when querying back series using the experimental blocks storage. #2400 [BUGFIX] Fixed collection of tracing spans from Thanos components used internally. #2655 [BUGFIX] Experimental TSDB: fixed memory leak in ingesters. #2586 [BUGFIX] QueryFrontend: fixed a situation where HTTP error is ignored and an incorrect status code is set. #2590 [BUGFIX] Ingester: Fix an ingester starting up in the JOINING state and staying there forever. #2565 [BUGFIX] QueryFrontend: fixed a panic (integer divide by zero) in the query-frontend. The query-frontend now requires the -querier.default-evaluation-interval config to be set to the same value of the querier. #2614 [BUGFIX] Experimental TSDB: when the querier receives a /series request with a time range older than the data stored in the ingester, it now ignores the requested time range and returns known series anyway instead of returning an empty response. This aligns the behaviour with the chunks storage. #2617 [BUGFIX] Cassandra: fixed an edge case leading to an invalid CQL query when querying the index on a Cassandra store. #2639 [BUGFIX] Ingester: increment series per metric when recovering from WAL or transfer. #2674 [BUGFIX] Fixed wrong number of arguments for 'mget' command Redis error when a query has no chunks to lookup from storage. #2700 #2796 [BUGFIX] Ingester: Automatically remove old tmp checkpoints, fixing a potential disk space leak after an ingester crashes. #2726 1.1.0 / 2020-05-21 This release brings the usual mix of bugfixes and improvements. The biggest change is that WAL support for chunks is now considered to be production-ready!\nPlease make sure to review renamed metrics, and update your dashboards and alerts accordingly.\n[CHANGE] Added v1 API routes documented in #2327. #2372 Added -http.alertmanager-http-prefix flag which allows the configuration of the path where the Alertmanager API and UI can be reached. The default is set to /alertmanager. Added -http.prometheus-http-prefix flag which allows the configuration of the path where the Prometheus API and UI can be reached. The default is set to /prometheus. Updated the index hosted at the root prefix to point to the updated routes. Legacy routes hardcoded with the /api/prom prefix now respect the -http.prefix flag. [CHANGE] The metrics cortex_distributor_ingester_appends_total and distributor_ingester_append_failures_total now include a type label to differentiate between samples and metadata. #2336 [CHANGE] The metrics for number of chunks and bytes flushed to the chunk store are renamed. Note that previous metrics were counted pre-deduplication, while new metrics are counted after deduplication. #2463 cortex_ingester_chunks_stored_total \u003e cortex_chunk_store_stored_chunks_total cortex_ingester_chunk_stored_bytes_total \u003e cortex_chunk_store_stored_chunk_bytes_total [CHANGE] Experimental TSDB: renamed blocks meta fetcher metrics: #2375 cortex_querier_bucket_store_blocks_meta_syncs_total \u003e cortex_querier_blocks_meta_syncs_total cortex_querier_bucket_store_blocks_meta_sync_failures_total \u003e cortex_querier_blocks_meta_sync_failures_total cortex_querier_bucket_store_blocks_meta_sync_duration_seconds \u003e cortex_querier_blocks_meta_sync_duration_seconds cortex_querier_bucket_store_blocks_meta_sync_consistency_delay_seconds \u003e cortex_querier_blocks_meta_sync_consistency_delay_seconds [CHANGE] Experimental TSDB: Modified default values for compactor.deletion-delay option from 48h to 12h and -experimental.tsdb.bucket-store.ignore-deletion-marks-delay from 24h to 6h. #2414 [CHANGE] WAL: Default value of -ingester.checkpoint-enabled changed to true. #2416 [CHANGE] trace_id field in log files has been renamed to traceID. #2518 [CHANGE] Slow query log has a different output now. Previously used url field has been replaced with host and path, and query parameters are logged as individual log fields with qs_ prefix. #2520 [CHANGE] WAL: WAL and checkpoint compression is now disabled. #2436 [CHANGE] Update in dependency go-kit/kit from v0.9.0 to v0.10.0. HTML escaping disabled in JSON Logger. #2535 [CHANGE] Experimental TSDB: Removed cortex_\u003cservice\u003e_ prefix from Thanos objstore metrics and added component label to distinguish which Cortex component is doing API calls to the object storage when running in single-binary mode: #2568 cortex_\u003cservice\u003e_thanos_objstore_bucket_operations_total renamed to thanos_objstore_bucket_operations_total{component=\"\u003cname\u003e\"} cortex_\u003cservice\u003e_thanos_objstore_bucket_operation_failures_total renamed to thanos_objstore_bucket_operation_failures_total{component=\"\u003cname\u003e\"} cortex_\u003cservice\u003e_thanos_objstore_bucket_operation_duration_seconds renamed to thanos_objstore_bucket_operation_duration_seconds{component=\"\u003cname\u003e\"} cortex_\u003cservice\u003e_thanos_objstore_bucket_last_successful_upload_time renamed to thanos_objstore_bucket_last_successful_upload_time{component=\"\u003cname\u003e\"} [CHANGE] FIFO cache: The -\u003cprefix\u003e.fifocache.size CLI flag has been renamed to -\u003cprefix\u003e.fifocache.max-size-items as well as its YAML config option size renamed to max_size_items. #2319 [FEATURE] Ruler: The -ruler.evaluation-delay flag was added to allow users to configure a default evaluation delay for all rules in cortex. The default value is 0 which is the current behavior. #2423 [FEATURE] Experimental: Added a new object storage client for OpenStack Swift. #2440 [FEATURE] TLS config options added to the Server. #2535 [FEATURE] Experimental: Added support for /api/v1/metadata Prometheus-based endpoint. #2549 [FEATURE] Add ability to limit concurrent queries to Cassandra with -cassandra.query-concurrency flag. #2562 [FEATURE] Experimental TSDB: Introduced store-gateway service used by the experimental blocks storage to load and query blocks. The store-gateway optionally supports blocks sharding and replication via a dedicated hash ring, configurable via -experimental.store-gateway.sharding-enabled and -experimental.store-gateway.sharding-ring.* flags. The following metrics have been added: #2433 #2458 #2469 #2523 cortex_querier_storegateway_instances_hit_per_query [ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing cortex_discarded_samples_total metric. #2370 [ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. #2383 [ENHANCEMENT] Experimental TSDB: Added cortex_querier_blocks_meta_synced, which reflects current state of synced blocks over all tenants. #2392 [ENHANCEMENT] Added cortex_distributor_latest_seen_sample_timestamp_seconds metric to see how far behind Prometheus servers are in sending data. #2371 [ENHANCEMENT] FIFO cache to support eviction based on memory usage. Added -\u003cprefix\u003e.fifocache.max-size-bytes CLI flag and YAML config option max_size_bytes to specify memory limit of the cache. #2319, #2527 [ENHANCEMENT] Added -querier.worker-match-max-concurrent. Force worker concurrency to match the -querier.max-concurrent option. Overrides -querier.worker-parallelism. #2456 [ENHANCEMENT] Added the following metrics for monitoring delete requests: #2445 cortex_purger_delete_requests_received_total: Number of delete requests received per user. cortex_purger_delete_requests_processed_total: Number of delete requests processed per user. cortex_purger_delete_requests_chunks_selected_total: Number of chunks selected while building delete plans per user. cortex_purger_delete_requests_processing_failures_total: Number of delete requests processing failures per user. [ENHANCEMENT] Single Binary: Added query-frontend to the single binary. Single binary users will now benefit from various query-frontend features. Primarily: sharding, parallelization, load shedding, additional caching (if configured), and query retries. #2437 [ENHANCEMENT] Allow 1w (where w denotes week) and 1y (where y denotes year) when setting -store.cache-lookups-older-than and -store.max-look-back-period. #2454 [ENHANCEMENT] Optimize index queries for matchers using “a|b|c”-type regex. #2446 #2475 [ENHANCEMENT] Added per tenant metrics for queries and chunks and bytes read from chunk store: #2463 cortex_chunk_store_fetched_chunks_total and cortex_chunk_store_fetched_chunk_bytes_total cortex_query_frontend_queries_total (per tenant queries counted by the frontend) [ENHANCEMENT] WAL: New metrics cortex_ingester_wal_logged_bytes_total and cortex_ingester_checkpoint_logged_bytes_total added to track total bytes logged to disk for WAL and checkpoints. #2497 [ENHANCEMENT] Add de-duplicated chunks counter cortex_chunk_store_deduped_chunks_total which counts every chunk not sent to the store because it was already sent by another replica. #2485 [ENHANCEMENT] Query-frontend now also logs the POST data of long queries. #2481 [ENHANCEMENT] WAL: Ingester WAL records now have type header and the custom WAL records have been replaced by Prometheus TSDB’s WAL records. Old records will not be supported from 1.3 onwards. Note: once this is deployed, you cannot downgrade without data loss. #2436 [ENHANCEMENT] Redis Cache: Added idle_timeout, wait_on_pool_exhaustion and max_conn_lifetime options to redis cache configuration. #2550 [ENHANCEMENT] WAL: the experimental tag has been removed on the WAL in ingesters. #2560 [ENHANCEMENT] Use newer AWS API for paginated queries - removes ‘Deprecated’ message from logfiles. #2452 [ENHANCEMENT] Experimental memberlist: Add retry with backoff on memberlist join other members. #2705 [ENHANCEMENT] Experimental TSDB: when the store-gateway sharding is enabled, unhealthy store-gateway instances are automatically removed from the ring after 10 consecutive -experimental.store-gateway.sharding-ring.heartbeat-timeout periods. #2526 [BUGFIX] Ruler: Ensure temporary rule files with special characters are properly mapped and cleaned up. #2506 [BUGFIX] Ensure requests are properly routed to the prometheus api embedded in the query if -server.path-prefix is set. Fixes #2411. #2372 [BUGFIX] Experimental TSDB: Fixed chunk data corruption when querying back series using the experimental blocks storage. #2400 [BUGFIX] Cassandra Storage: Fix endpoint TLS host verification. #2109 [BUGFIX] Experimental TSDB: Fixed response status code from 422 to 500 when an error occurs while iterating chunks with the experimental blocks storage. #2402 [BUGFIX] Ring: Fixed a situation where upgrading from pre-1.0 cortex with a rolling strategy caused new 1.0 ingesters to lose their zone value in the ring until manually forced to re-register. #2404 [BUGFIX] Distributor: /all_user_stats now show API and Rule Ingest Rate correctly. #2457 [BUGFIX] Fixed version, revision and branch labels exported by the cortex_build_info metric. #2468 [BUGFIX] QueryFrontend: fixed a situation where span context missed when downstream_url is used. #2539 [BUGFIX] Querier: Fixed a situation where querier would crash because of an unresponsive frontend instance. #2569 1.0.1 / 2020-04-23 [BUGFIX] Fix gaps when querying ingesters with replication factor = 3 and 2 ingesters in the cluster. #2503 1.0.0 / 2020-04-02 This is the first major release of Cortex. We made a lot of breaking changes in this release which have been detailed below. Please also see the stability guarantees we provide as part of a major release: https://cortexmetrics.io/docs/configuration/v1guarantees/\n[CHANGE] Remove the following deprecated flags: #2339\n-metrics.error-rate-query (use -metrics.write-throttle-query instead). -store.cardinality-cache-size (use -store.index-cache-read.enable-fifocache and -store.index-cache-read.fifocache.size instead). -store.cardinality-cache-validity (use -store.index-cache-read.enable-fifocache and -store.index-cache-read.fifocache.duration instead). -distributor.limiter-reload-period (flag unused) -ingester.claim-on-rollout (flag unused) -ingester.normalise-tokens (flag unused) [CHANGE] Renamed YAML file options to be more consistent. See full config file changes below. #2273\n[CHANGE] AWS based autoscaling has been removed. You can only use metrics based autoscaling now. -applicationautoscaling.url has been removed. See https://cortexmetrics.io/docs/production/aws/#dynamodb-capacity-provisioning on how to migrate. #2328\n[CHANGE] Renamed the memcache.write-back-goroutines and memcache.write-back-buffer flags to background.write-back-concurrency and background.write-back-buffer. This affects the following flags: #2241\n-frontend.memcache.write-back-buffer –\u003e -frontend.background.write-back-buffer -frontend.memcache.write-back-goroutines –\u003e -frontend.background.write-back-concurrency -store.index-cache-read.memcache.write-back-buffer –\u003e -store.index-cache-read.background.write-back-buffer -store.index-cache-read.memcache.write-back-goroutines –\u003e -store.index-cache-read.background.write-back-concurrency -store.index-cache-write.memcache.write-back-buffer –\u003e -store.index-cache-write.background.write-back-buffer -store.index-cache-write.memcache.write-back-goroutines –\u003e -store.index-cache-write.background.write-back-concurrency -memcache.write-back-buffer –\u003e -store.chunks-cache.background.write-back-buffer. Note the next change log for the difference. -memcache.write-back-goroutines –\u003e -store.chunks-cache.background.write-back-concurrency. Note the next change log for the difference. [CHANGE] Renamed the chunk cache flags to have store.chunks-cache. as prefix. This means the following flags have been changed: #2241\n-cache.enable-fifocache –\u003e -store.chunks-cache.cache.enable-fifocache -default-validity –\u003e -store.chunks-cache.default-validity -fifocache.duration –\u003e -store.chunks-cache.fifocache.duration -fifocache.size –\u003e -store.chunks-cache.fifocache.size -memcache.write-back-buffer –\u003e -store.chunks-cache.background.write-back-buffer. Note the previous change log for the difference. -memcache.write-back-goroutines –\u003e -store.chunks-cache.background.write-back-concurrency. Note the previous change log for the difference. -memcached.batchsize –\u003e -store.chunks-cache.memcached.batchsize -memcached.consistent-hash –\u003e -store.chunks-cache.memcached.consistent-hash -memcached.expiration –\u003e -store.chunks-cache.memcached.expiration -memcached.hostname –\u003e -store.chunks-cache.memcached.hostname -memcached.max-idle-conns –\u003e -store.chunks-cache.memcached.max-idle-conns -memcached.parallelism –\u003e -store.chunks-cache.memcached.parallelism -memcached.service –\u003e -store.chunks-cache.memcached.service -memcached.timeout –\u003e -store.chunks-cache.memcached.timeout -memcached.update-interval –\u003e -store.chunks-cache.memcached.update-interval -redis.enable-tls –\u003e -store.chunks-cache.redis.enable-tls -redis.endpoint –\u003e -store.chunks-cache.redis.endpoint -redis.expiration –\u003e -store.chunks-cache.redis.expiration -redis.max-active-conns –\u003e -store.chunks-cache.redis.max-active-conns -redis.max-idle-conns –\u003e -store.chunks-cache.redis.max-idle-conns -redis.password –\u003e -store.chunks-cache.redis.password -redis.timeout –\u003e -store.chunks-cache.redis.timeout [CHANGE] Rename the -store.chunk-cache-stubs to -store.chunks-cache.cache-stubs to be more inline with above. #2241\n[CHANGE] Change prefix of flags -dynamodb.periodic-table.* to -table-manager.index-table.*. #2359\n[CHANGE] Change prefix of flags -dynamodb.chunk-table.* to -table-manager.chunk-table.*. #2359\n[CHANGE] Change the following flags: #2359\n-dynamodb.poll-interval –\u003e -table-manager.poll-interval -dynamodb.periodic-table.grace-period –\u003e -table-manager.periodic-table.grace-period [CHANGE] Renamed the following flags: #2273\n-dynamodb.chunk.gang.size –\u003e -dynamodb.chunk-gang-size -dynamodb.chunk.get.max.parallelism –\u003e -dynamodb.chunk-get-max-parallelism [CHANGE] Don’t support mixed time units anymore for duration. For example, 168h5m0s doesn’t work anymore, please use just one unit (s|m|h|d|w|y). #2252\n[CHANGE] Utilize separate protos for rule state and storage. Experimental ruler API will not be functional until the rollout is complete. #2226\n[CHANGE] Frontend worker in querier now starts after all Querier module dependencies are started. This fixes issue where frontend worker started to send queries to querier before it was ready to serve them (mostly visible when using experimental blocks storage). #2246\n[CHANGE] Lifecycler component now enters Failed state on errors, and doesn’t exit the process. (Important if you’re vendoring Cortex and use Lifecycler) #2251\n[CHANGE] /ready handler now returns 200 instead of 204. #2330\n[CHANGE] Better defaults for the following options: #2344\n-\u003cprefix\u003e.consul.consistent-reads: Old default: true, new default: false. This reduces the load on Consul. -\u003cprefix\u003e.consul.watch-rate-limit: Old default: 0, new default: 1. This rate limits the reads to 1 per second. Which is good enough for ring watches. -distributor.health-check-ingesters: Old default: false, new default: true. -ingester.max-stale-chunk-idle: Old default: 0, new default: 2m. This lets us expire series that we know are stale early. -ingester.spread-flushes: Old default: false, new default: true. This allows to better de-duplicate data and use less space. -ingester.chunk-age-jitter: Old default: 20mins, new default: 0. This is to enable the -ingester.spread-flushes to true. -\u003cprefix\u003e.memcached.batchsize: Old default: 0, new default: 1024. This allows batching of requests and keeps the concurrent requests low. -\u003cprefix\u003e.memcached.consistent-hash: Old default: false, new default: true. This allows for better cache hits when the memcaches are scaled up and down. -querier.batch-iterators: Old default: false, new default: true. -querier.ingester-streaming: Old default: false, new default: true. [CHANGE] Experimental TSDB: Added -experimental.tsdb.bucket-store.postings-cache-compression-enabled to enable postings compression when storing to cache. #2335\n[CHANGE] Experimental TSDB: Added -compactor.deletion-delay, which is time before a block marked for deletion is deleted from bucket. If not 0, blocks will be marked for deletion and compactor component will delete blocks marked for deletion from the bucket. If delete-delay is 0, blocks will be deleted straight away. Note that deleting blocks immediately can cause query failures, if store gateway / querier still has the block loaded, or compactor is ignoring the deletion because it’s compacting the block at the same time. Default value is 48h. #2335\n[CHANGE] Experimental TSDB: Added -experimental.tsdb.bucket-store.index-cache.postings-compression-enabled, to set duration after which the blocks marked for deletion will be filtered out while fetching blocks used for querying. This option allows querier to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet. Default is 24h, half of the default value for -compactor.deletion-delay. #2335\n[CHANGE] Experimental TSDB: Added -experimental.tsdb.bucket-store.index-cache.memcached.max-item-size to control maximum size of item that is stored to memcached. Defaults to 1 MiB. #2335\n[FEATURE] Added experimental storage API to the ruler service that is enabled when the -experimental.ruler.enable-api is set to true #2269\n-ruler.storage.type flag now allows s3,gcs, and azure values -ruler.storage.(s3|gcs|azure) flags exist to allow the configuration of object clients set for rule storage [CHANGE] Renamed table manager metrics. #2307 #2359\ncortex_dynamo_sync_tables_seconds -\u003e cortex_table_manager_sync_duration_seconds cortex_dynamo_table_capacity_units -\u003e cortex_table_capacity_units [FEATURE] Flusher target to flush the WAL. #2075\n-flusher.wal-dir for the WAL directory to recover from. -flusher.concurrent-flushes for number of concurrent flushes. -flusher.flush-op-timeout is duration after which a flush should timeout. [FEATURE] Ingesters can now have an optional availability zone set, to ensure metric replication is distributed across zones. This is set via the -ingester.availability-zone flag or the availability_zone field in the config file. #2317\n[ENHANCEMENT] Better reuse of connections to DynamoDB and S3. #2268\n[ENHANCEMENT] Reduce number of goroutines used while executing a single index query. #2280\n[ENHANCEMENT] Experimental TSDB: Add support for local filesystem backend. #2245\n[ENHANCEMENT] Experimental TSDB: Added memcached support for the TSDB index cache. #2290\n[ENHANCEMENT] Experimental TSDB: Removed gRPC server to communicate between querier and BucketStore. #2324\n[ENHANCEMENT] Allow 1w (where w denotes week) and 1y (where y denotes year) when setting table period and retention. #2252\n[ENHANCEMENT] Added FIFO cache metrics for current number of entries and memory usage. #2270\n[ENHANCEMENT] Output all config fields to /config API, including those with empty value. #2209\n[ENHANCEMENT] Add “missing_metric_name” and “metric_name_invalid” reasons to cortex_discarded_samples_total metric. #2346\n[ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing cortex_discarded_samples_total metric. #2370\n[BUGFIX] Ensure user state metrics are updated if a transfer fails. #2338\n[BUGFIX] Fixed etcd client keepalive settings. #2278\n[BUGFIX] Register the metrics of the WAL. #2295\n[BUXFIX] Experimental TSDB: fixed error handling when ingesting out of bound samples. #2342\nKnown issues This experimental blocks storage in Cortex 1.0.0 has a bug which may lead to the error cannot iterate chunk for series when running queries. This bug has been fixed in #2400. If you’re running the experimental blocks storage, please build Cortex from master. Config file breaking changes In this section you can find a config file diff showing the breaking changes introduced in Cortex. You can also find the full configuration file reference doc in the website.\n### ingester_config # Period with which to attempt to flush chunks. # CLI flag: -ingester.flush-period -[flushcheckperiod: \u003cduration\u003e | default = 1m0s] +[flush_period: \u003cduration\u003e | default = 1m0s] # Period chunks will remain in memory after flushing. # CLI flag: -ingester.retain-period -[retainperiod: \u003cduration\u003e | default = 5m0s] +[retain_period: \u003cduration\u003e | default = 5m0s] # Maximum chunk idle time before flushing. # CLI flag: -ingester.max-chunk-idle -[maxchunkidle: \u003cduration\u003e | default = 5m0s] +[max_chunk_idle_time: \u003cduration\u003e | default = 5m0s] # Maximum chunk idle time for chunks terminating in stale markers before # flushing. 0 disables it and a stale series is not flushed until the # max-chunk-idle timeout is reached. # CLI flag: -ingester.max-stale-chunk-idle -[maxstalechunkidle: \u003cduration\u003e | default = 0s] +[max_stale_chunk_idle_time: \u003cduration\u003e | default = 2m0s] # Timeout for individual flush operations. # CLI flag: -ingester.flush-op-timeout -[flushoptimeout: \u003cduration\u003e | default = 1m0s] +[flush_op_timeout: \u003cduration\u003e | default = 1m0s] # Maximum chunk age before flushing. # CLI flag: -ingester.max-chunk-age -[maxchunkage: \u003cduration\u003e | default = 12h0m0s] +[max_chunk_age: \u003cduration\u003e | default = 12h0m0s] -# Range of time to subtract from MaxChunkAge to spread out flushes +# Range of time to subtract from -ingester.max-chunk-age to spread out flushes # CLI flag: -ingester.chunk-age-jitter -[chunkagejitter: \u003cduration\u003e | default = 20m0s] +[chunk_age_jitter: \u003cduration\u003e | default = 0] # Number of concurrent goroutines flushing to dynamodb. # CLI flag: -ingester.concurrent-flushes -[concurrentflushes: \u003cint\u003e | default = 50] +[concurrent_flushes: \u003cint\u003e | default = 50] -# If true, spread series flushes across the whole period of MaxChunkAge +# If true, spread series flushes across the whole period of +# -ingester.max-chunk-age. # CLI flag: -ingester.spread-flushes -[spreadflushes: \u003cboolean\u003e | default = false] +[spread_flushes: \u003cboolean\u003e | default = true] # Period with which to update the per-user ingestion rates. # CLI flag: -ingester.rate-update-period -[rateupdateperiod: \u003cduration\u003e | default = 15s] +[rate_update_period: \u003cduration\u003e | default = 15s] ### querier_config # The maximum number of concurrent queries. # CLI flag: -querier.max-concurrent -[maxconcurrent: \u003cint\u003e | default = 20] +[max_concurrent: \u003cint\u003e | default = 20] # Use batch iterators to execute query, as opposed to fully materialising the # series in memory. Takes precedent over the -querier.iterators flag. # CLI flag: -querier.batch-iterators -[batchiterators: \u003cboolean\u003e | default = false] +[batch_iterators: \u003cboolean\u003e | default = true] # Use streaming RPCs to query ingester. # CLI flag: -querier.ingester-streaming -[ingesterstreaming: \u003cboolean\u003e | default = false] +[ingester_streaming: \u003cboolean\u003e | default = true] # Maximum number of samples a single query can load into memory. # CLI flag: -querier.max-samples -[maxsamples: \u003cint\u003e | default = 50000000] +[max_samples: \u003cint\u003e | default = 50000000] # The default evaluation interval or step size for subqueries. # CLI flag: -querier.default-evaluation-interval -[defaultevaluationinterval: \u003cduration\u003e | default = 1m0s] +[default_evaluation_interval: \u003cduration\u003e | default = 1m0s] ### query_frontend_config # URL of downstream Prometheus. # CLI flag: -frontend.downstream-url -[downstream: \u003cstring\u003e | default = \"\"] +[downstream_url: \u003cstring\u003e | default = \"\"] ### ruler_config # URL of alerts return path. # CLI flag: -ruler.external.url -[externalurl: \u003curl\u003e | default = ] +[external_url: \u003curl\u003e | default = ] # How frequently to evaluate rules # CLI flag: -ruler.evaluation-interval -[evaluationinterval: \u003cduration\u003e | default = 1m0s] +[evaluation_interval: \u003cduration\u003e | default = 1m0s] # How frequently to poll for rule changes # CLI flag: -ruler.poll-interval -[pollinterval: \u003cduration\u003e | default = 1m0s] +[poll_interval: \u003cduration\u003e | default = 1m0s] -storeconfig: +storage: # file path to store temporary rule files for the prometheus rule managers # CLI flag: -ruler.rule-path -[rulepath: \u003cstring\u003e | default = \"/rules\"] +[rule_path: \u003cstring\u003e | default = \"/rules\"] # URL of the Alertmanager to send notifications to. # CLI flag: -ruler.alertmanager-url -[alertmanagerurl: \u003curl\u003e | default = ] +[alertmanager_url: \u003curl\u003e | default = ] # Use DNS SRV records to discover alertmanager hosts. # CLI flag: -ruler.alertmanager-discovery -[alertmanagerdiscovery: \u003cboolean\u003e | default = false] +[enable_alertmanager_discovery: \u003cboolean\u003e | default = false] # How long to wait between refreshing alertmanager hosts. # CLI flag: -ruler.alertmanager-refresh-interval -[alertmanagerrefreshinterval: \u003cduration\u003e | default = 1m0s] +[alertmanager_refresh_interval: \u003cduration\u003e | default = 1m0s] # If enabled requests to alertmanager will utilize the V2 API. # CLI flag: -ruler.alertmanager-use-v2 -[alertmanangerenablev2api: \u003cboolean\u003e | default = false] +[enable_alertmanager_v2: \u003cboolean\u003e | default = false] # Capacity of the queue for notifications to be sent to the Alertmanager. # CLI flag: -ruler.notification-queue-capacity -[notificationqueuecapacity: \u003cint\u003e | default = 10000] +[notification_queue_capacity: \u003cint\u003e | default = 10000] # HTTP timeout duration when sending notifications to the Alertmanager. # CLI flag: -ruler.notification-timeout -[notificationtimeout: \u003cduration\u003e | default = 10s] +[notification_timeout: \u003cduration\u003e | default = 10s] # Distribute rule evaluation using ring backend # CLI flag: -ruler.enable-sharding -[enablesharding: \u003cboolean\u003e | default = false] +[enable_sharding: \u003cboolean\u003e | default = false] # Time to spend searching for a pending ruler when shutting down. # CLI flag: -ruler.search-pending-for -[searchpendingfor: \u003cduration\u003e | default = 5m0s] +[search_pending_for: \u003cduration\u003e | default = 5m0s] # Period with which to attempt to flush rule groups. # CLI flag: -ruler.flush-period -[flushcheckperiod: \u003cduration\u003e | default = 1m0s] +[flush_period: \u003cduration\u003e | default = 1m0s] ### alertmanager_config # Base path for data storage. # CLI flag: -alertmanager.storage.path -[datadir: \u003cstring\u003e | default = \"data/\"] +[data_dir: \u003cstring\u003e | default = \"data/\"] # will be used to prefix all HTTP endpoints served by Alertmanager. If omitted, # relevant URL components will be derived automatically. # CLI flag: -alertmanager.web.external-url -[externalurl: \u003curl\u003e | default = ] +[external_url: \u003curl\u003e | default = ] # How frequently to poll Cortex configs # CLI flag: -alertmanager.configs.poll-interval -[pollinterval: \u003cduration\u003e | default = 15s] +[poll_interval: \u003cduration\u003e | default = 15s] # Listen address for cluster. # CLI flag: -cluster.listen-address -[clusterbindaddr: \u003cstring\u003e | default = \"0.0.0.0:9094\"] +[cluster_bind_address: \u003cstring\u003e | default = \"0.0.0.0:9094\"] # Explicit address to advertise in cluster. # CLI flag: -cluster.advertise-address -[clusteradvertiseaddr: \u003cstring\u003e | default = \"\"] +[cluster_advertise_address: \u003cstring\u003e | default = \"\"] # Time to wait between peers to send notifications. # CLI flag: -cluster.peer-timeout -[peertimeout: \u003cduration\u003e | default = 15s] +[peer_timeout: \u003cduration\u003e | default = 15s] # Filename of fallback config to use if none specified for instance. # CLI flag: -alertmanager.configs.fallback -[fallbackconfigfile: \u003cstring\u003e | default = \"\"] +[fallback_config_file: \u003cstring\u003e | default = \"\"] # Root of URL to generate if config is http://internal.monitor # CLI flag: -alertmanager.configs.auto-webhook-root -[autowebhookroot: \u003cstring\u003e | default = \"\"] +[auto_webhook_root: \u003cstring\u003e | default = \"\"] ### table_manager_config -store: +storage: -# How frequently to poll DynamoDB to learn our capacity. -# CLI flag: -dynamodb.poll-interval -[dynamodb_poll_interval: \u003cduration\u003e | default = 2m0s] +# How frequently to poll backend to learn our capacity. +# CLI flag: -table-manager.poll-interval +[poll_interval: \u003cduration\u003e | default = 2m0s] -# DynamoDB periodic tables grace period (duration which table will be -# created/deleted before/after it's needed). -# CLI flag: -dynamodb.periodic-table.grace-period +# Periodic tables grace period (duration which table will be created/deleted +# before/after it's needed). +# CLI flag: -table-manager.periodic-table.grace-period [creation_grace_period: \u003cduration\u003e | default = 10m0s] index_tables_provisioning: # Enables on demand throughput provisioning for the storage provider (if - # supported). Applies only to tables which are not autoscaled - # CLI flag: -dynamodb.periodic-table.enable-ondemand-throughput-mode - [provisioned_throughput_on_demand_mode: \u003cboolean\u003e | default = false] + # supported). Applies only to tables which are not autoscaled. Supported by + # DynamoDB + # CLI flag: -table-manager.index-table.enable-ondemand-throughput-mode + [enable_ondemand_throughput_mode: \u003cboolean\u003e | default = false] # Enables on demand throughput provisioning for the storage provider (if - # supported). Applies only to tables which are not autoscaled - # CLI flag: -dynamodb.periodic-table.inactive-enable-ondemand-throughput-mode - [inactive_throughput_on_demand_mode: \u003cboolean\u003e | default = false] + # supported). Applies only to tables which are not autoscaled. Supported by + # DynamoDB + # CLI flag: -table-manager.index-table.inactive-enable-ondemand-throughput-mode + [enable_inactive_throughput_on_demand_mode: \u003cboolean\u003e | default = false] chunk_tables_provisioning: # Enables on demand throughput provisioning for the storage provider (if - # supported). Applies only to tables which are not autoscaled - # CLI flag: -dynamodb.chunk-table.enable-ondemand-throughput-mode - [provisioned_throughput_on_demand_mode: \u003cboolean\u003e | default = false] + # supported). Applies only to tables which are not autoscaled. Supported by + # DynamoDB + # CLI flag: -table-manager.chunk-table.enable-ondemand-throughput-mode + [enable_ondemand_throughput_mode: \u003cboolean\u003e | default = false] ### storage_config aws: - dynamodbconfig: + dynamodb: # DynamoDB endpoint URL with escaped Key and Secret encoded. If only region # is specified as a host, proper endpoint will be deduced. Use # inmemory:///\u003ctable-name\u003e to use a mock in-memory implementation. # CLI flag: -dynamodb.url - [dynamodb: \u003curl\u003e | default = ] + [dynamodb_url: \u003curl\u003e | default = ] # DynamoDB table management requests per second limit. # CLI flag: -dynamodb.api-limit - [apilimit: \u003cfloat\u003e | default = 2] + [api_limit: \u003cfloat\u003e | default = 2] # DynamoDB rate cap to back off when throttled. # CLI flag: -dynamodb.throttle-limit - [throttlelimit: \u003cfloat\u003e | default = 10] + [throttle_limit: \u003cfloat\u003e | default = 10] - - # ApplicationAutoscaling endpoint URL with escaped Key and Secret encoded. - # CLI flag: -applicationautoscaling.url - [applicationautoscaling: \u003curl\u003e | default = ] # Queue length above which we will scale up capacity # CLI flag: -metrics.target-queue-length - [targetqueuelen: \u003cint\u003e | default = 100000] + [target_queue_length: \u003cint\u003e | default = 100000] # Scale up capacity by this multiple # CLI flag: -metrics.scale-up-factor - [scaleupfactor: \u003cfloat\u003e | default = 1.3] + [scale_up_factor: \u003cfloat\u003e | default = 1.3] # Ignore throttling below this level (rate per second) # CLI flag: -metrics.ignore-throttle-below - [minthrottling: \u003cfloat\u003e | default = 1] + [ignore_throttle_below: \u003cfloat\u003e | default = 1] # query to fetch ingester queue length # CLI flag: -metrics.queue-length-query - [queuelengthquery: \u003cstring\u003e | default = \"sum(avg_over_time(cortex_ingester_flush_queue_length{job=\\\"cortex/ingester\\\"}[2m]))\"] + [queue_length_query: \u003cstring\u003e | default = \"sum(avg_over_time(cortex_ingester_flush_queue_length{job=\\\"cortex/ingester\\\"}[2m]))\"] # query to fetch throttle rates per table # CLI flag: -metrics.write-throttle-query - [throttlequery: \u003cstring\u003e | default = \"sum(rate(cortex_dynamo_throttled_total{operation=\\\"DynamoDB.BatchWriteItem\\\"}[1m])) by (table) \u003e 0\"] + [write_throttle_query: \u003cstring\u003e | default = \"sum(rate(cortex_dynamo_throttled_total{operation=\\\"DynamoDB.BatchWriteItem\\\"}[1m])) by (table) \u003e 0\"] # query to fetch write capacity usage per table # CLI flag: -metrics.usage-query - [usagequery: \u003cstring\u003e | default = \"sum(rate(cortex_dynamo_consumed_capacity_total{operation=\\\"DynamoDB.BatchWriteItem\\\"}[15m])) by (table) \u003e 0\"] + [write_usage_query: \u003cstring\u003e | default = \"sum(rate(cortex_dynamo_consumed_capacity_total{operation=\\\"DynamoDB.BatchWriteItem\\\"}[15m])) by (table) \u003e 0\"] # query to fetch read capacity usage per table # CLI flag: -metrics.read-usage-query - [readusagequery: \u003cstring\u003e | default = \"sum(rate(cortex_dynamo_consumed_capacity_total{operation=\\\"DynamoDB.QueryPages\\\"}[1h])) by (table) \u003e 0\"] + [read_usage_query: \u003cstring\u003e | default = \"sum(rate(cortex_dynamo_consumed_capacity_total{operation=\\\"DynamoDB.QueryPages\\\"}[1h])) by (table) \u003e 0\"] # query to fetch read errors per table # CLI flag: -metrics.read-error-query - [readerrorquery: \u003cstring\u003e | default = \"sum(increase(cortex_dynamo_failures_total{operation=\\\"DynamoDB.QueryPages\\\",error=\\\"ProvisionedThroughputExceededException\\\"}[1m])) by (table) \u003e 0\"] + [read_error_query: \u003cstring\u003e | default = \"sum(increase(cortex_dynamo_failures_total{operation=\\\"DynamoDB.QueryPages\\\",error=\\\"ProvisionedThroughputExceededException\\\"}[1m])) by (table) \u003e 0\"] # Number of chunks to group together to parallelise fetches (zero to # disable) - # CLI flag: -dynamodb.chunk.gang.size - [chunkgangsize: \u003cint\u003e | default = 10] + # CLI flag: -dynamodb.chunk-gang-size + [chunk_gang_size: \u003cint\u003e | default = 10] # Max number of chunk-get operations to start in parallel - # CLI flag: -dynamodb.chunk.get.max.parallelism - [chunkgetmaxparallelism: \u003cint\u003e | default = 32] + # CLI flag: -dynamodb.chunk.get-max-parallelism + [chunk_get_max_parallelism: \u003cint\u003e | default = 32] backoff_config: # Minimum delay when backing off. # CLI flag: -bigtable.backoff-min-period - [minbackoff: \u003cduration\u003e | default = 100ms] + [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -bigtable.backoff-max-period - [maxbackoff: \u003cduration\u003e | default = 10s] + [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -bigtable.backoff-retries - [maxretries: \u003cint\u003e | default = 10] + [max_retries: \u003cint\u003e | default = 10] # If enabled, once a tables info is fetched, it is cached. # CLI flag: -bigtable.table-cache.enabled - [tablecacheenabled: \u003cboolean\u003e | default = true] + [table_cache_enabled: \u003cboolean\u003e | default = true] # Duration to cache tables before checking again. # CLI flag: -bigtable.table-cache.expiration - [tablecacheexpiration: \u003cduration\u003e | default = 30m0s] + [table_cache_expiration: \u003cduration\u003e | default = 30m0s] # Cache validity for active index entries. Should be no higher than # -ingester.max-chunk-idle. # CLI flag: -store.index-cache-validity -[indexcachevalidity: \u003cduration\u003e | default = 5m0s] +[index_cache_validity: \u003cduration\u003e | default = 5m0s] ### ingester_client_config grpc_client_config: backoff_config: # Minimum delay when backing off. # CLI flag: -ingester.client.backoff-min-period - [minbackoff: \u003cduration\u003e | default = 100ms] + [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -ingester.client.backoff-max-period - [maxbackoff: \u003cduration\u003e | default = 10s] + [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -ingester.client.backoff-retries - [maxretries: \u003cint\u003e | default = 10] + [max_retries: \u003cint\u003e | default = 10] ### frontend_worker_config -# Address of query frontend service. +# Address of query frontend service, in host:port format. # CLI flag: -querier.frontend-address -[address: \u003cstring\u003e | default = \"\"] +[frontend_address: \u003cstring\u003e | default = \"\"] # How often to query DNS. # CLI flag: -querier.dns-lookup-period -[dnslookupduration: \u003cduration\u003e | default = 10s] +[dns_lookup_duration: \u003cduration\u003e | default = 10s] grpc_client_config: backoff_config: # Minimum delay when backing off. # CLI flag: -querier.frontend-client.backoff-min-period - [minbackoff: \u003cduration\u003e | default = 100ms] + [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -querier.frontend-client.backoff-max-period - [maxbackoff: \u003cduration\u003e | default = 10s] + [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -querier.frontend-client.backoff-retries - [maxretries: \u003cint\u003e | default = 10] + [max_retries: \u003cint\u003e | default = 10] ### consul_config # ACL Token used to interact with Consul. -# CLI flag: -\u003cprefix\u003e.consul.acltoken -[acltoken: \u003cstring\u003e | default = \"\"] +# CLI flag: -\u003cprefix\u003e.consul.acl-token +[acl_token: \u003cstring\u003e | default = \"\"] # HTTP timeout when talking to Consul # CLI flag: -\u003cprefix\u003e.consul.client-timeout -[httpclienttimeout: \u003cduration\u003e | default = 20s] +[http_client_timeout: \u003cduration\u003e | default = 20s] # Enable consistent reads to Consul. # CLI flag: -\u003cprefix\u003e.consul.consistent-reads -[consistentreads: \u003cboolean\u003e | default = true] +[consistent_reads: \u003cboolean\u003e | default = false] # Rate limit when watching key or prefix in Consul, in requests per second. 0 # disables the rate limit. # CLI flag: -\u003cprefix\u003e.consul.watch-rate-limit -[watchkeyratelimit: \u003cfloat\u003e | default = 0] +[watch_rate_limit: \u003cfloat\u003e | default = 1] # Burst size used in rate limit. Values less than 1 are treated as 1. # CLI flag: -\u003cprefix\u003e.consul.watch-burst-size -[watchkeyburstsize: \u003cint\u003e | default = 1] +[watch_burst_size: \u003cint\u003e | default = 1] ### configstore_config # URL of configs API server. # CLI flag: -\u003cprefix\u003e.configs.url -[configsapiurl: \u003curl\u003e | default = ] +[configs_api_url: \u003curl\u003e | default = ] # Timeout for requests to Weave Cloud configs service. # CLI flag: -\u003cprefix\u003e.configs.client-timeout -[clienttimeout: \u003cduration\u003e | default = 5s] +[client_timeout: \u003cduration\u003e | default = 5s] 0.7.0 / 2020-03-16 Cortex 0.7.0 is a major step forward the upcoming 1.0 release. In this release, we’ve got 164 contributions from 26 authors. Thanks to all contributors! ❤️\nPlease be aware that Cortex 0.7.0 introduces some breaking changes. You’re encouraged to read all the [CHANGE] entries below before upgrading your Cortex cluster. In particular:\nCleaned up some configuration options in preparation for the Cortex 1.0.0 release (see also the annotated config file breaking changes below): Removed CLI flags support to configure the schema (see how to migrate from flags to schema file) Renamed CLI flag -config-yaml to -schema-config-file Removed CLI flag -store.min-chunk-age in favor of -querier.query-store-after. The corresponding YAML config option ingestermaxquerylookback has been renamed to query_ingesters_within Deprecated CLI flag -frontend.cache-split-interval in favor of -querier.split-queries-by-interval Renamed the YAML config option defaul_validity to default_validity Removed the YAML config option config_store (in the alertmanager YAML config) in favor of store Removed the YAML config root block configdb in favor of configs. This change is also reflected in the following CLI flags renaming: -database.* -\u003e -configs.database.* -database.migrations -\u003e -configs.database.migrations-dir Removed the fluentd-based billing infrastructure including the CLI flags: -distributor.enable-billing -billing.max-buffered-events -billing.retry-delay -billing.ingester Removed support for using denormalised tokens in the ring. Before upgrading, make sure your Cortex cluster is already running v0.6.0 or an earlier version with -ingester.normalise-tokens=true Full changelog [CHANGE] Removed support for flags to configure schema. Further, the flag for specifying the config file (-config-yaml) has been deprecated. Please use -schema-config-file. See the Schema Configuration documentation for more details on how to configure the schema using the YAML file. #2221 [CHANGE] In the config file, the root level config_store config option has been moved to alertmanager \u003e store \u003e configdb. #2125 [CHANGE] Removed unnecessary frontend.cache-split-interval in favor of querier.split-queries-by-interval both to reduce configuration complexity and guarantee alignment of these two configs. Starting from now, -querier.cache-results may only be enabled in conjunction with -querier.split-queries-by-interval (previously the cache interval default was 24h so if you want to preserve the same behaviour you should set -querier.split-queries-by-interval=24h). #2040 [CHANGE] Renamed Configs configuration options. #2187 configuration options -database.* -\u003e -configs.database.* -database.migrations -\u003e -configs.database.migrations-dir config file configdb.uri: -\u003e configs.database.uri: configdb.migrationsdir: -\u003e configs.database.migrations_dir: configdb.passwordfile: -\u003e configs.database.password_file: [CHANGE] Moved -store.min-chunk-age to the Querier config as -querier.query-store-after, allowing the store to be skipped during query time if the metrics wouldn’t be found. The YAML config option ingestermaxquerylookback has been renamed to query_ingesters_within to match its CLI flag. #1893 [CHANGE] Renamed the cache configuration setting defaul_validity to default_validity. #2140 [CHANGE] Remove fluentd-based billing infrastructure and flags such as -distributor.enable-billing. #1491 [CHANGE] Removed remaining support for using denormalised tokens in the ring. If you’re still running ingesters with denormalised tokens (Cortex 0.4 or earlier, with -ingester.normalise-tokens=false), such ingesters will now be completely invisible to distributors and need to be either switched to Cortex 0.6.0 or later, or be configured to use normalised tokens. #2034 [CHANGE] The frontend http server will now send 502 in case of deadline exceeded and 499 if the user requested cancellation. #2156 [CHANGE] We now enforce queries to be up to -querier.max-query-into-future into the future (defaults to 10m). #1929 -store.min-chunk-age has been removed -querier.query-store-after has been added in it’s place. [CHANGE] Removed unused /validate_expr endpoint. #2152 [CHANGE] Updated Prometheus dependency to v2.16.0. This Prometheus version uses Active Query Tracker to limit concurrent queries. In order to keep -querier.max-concurrent working, Active Query Tracker is enabled by default, and is configured to store its data to active-query-tracker directory (relative to current directory when Cortex started). This can be changed by using -querier.active-query-tracker-dir option. Purpose of Active Query Tracker is to log queries that were running when Cortex crashes. This logging happens on next Cortex start. #2088 [CHANGE] Default to BigChunk encoding; may result in slightly higher disk usage if many timeseries have a constant value, but should generally result in fewer, bigger chunks. #2207 [CHANGE] WAL replays are now done while the rest of Cortex is starting, and more specifically, when HTTP server is running. This makes it possible to scrape metrics during WAL replays. Applies to both chunks and experimental blocks storage. #2222 [CHANGE] Cortex now has /ready probe for all services, not just ingester and querier as before. In single-binary mode, /ready reports 204 only if all components are running properly. #2166 [CHANGE] If you are vendoring Cortex and use its components in your project, be aware that many Cortex components no longer start automatically when they are created. You may want to review PR and attached document. #2166 [CHANGE] Experimental TSDB: the querier in-memory index cache used by the experimental blocks storage shifted from per-tenant to per-querier. The -experimental.tsdb.bucket-store.index-cache-size-bytes now configures the per-querier index cache max size instead of a per-tenant cache and its default has been increased to 1GB. #2189 [CHANGE] Experimental TSDB: TSDB head compaction interval and concurrency is now configurable (defaults to 1 min interval and 5 concurrent head compactions). New options: -experimental.tsdb.head-compaction-interval and -experimental.tsdb.head-compaction-concurrency. #2172 [CHANGE] Experimental TSDB: switched the blocks storage index header to the binary format. This change is expected to have no visible impact, except lower startup times and memory usage in the queriers. It’s possible to switch back to the old JSON format via the flag -experimental.tsdb.bucket-store.binary-index-header-enabled=false. #2223 [CHANGE] Experimental Memberlist KV store can now be used in single-binary Cortex. Attempts to use it previously would fail with panic. This change also breaks existing binary protocol used to exchange gossip messages, so this version will not be able to understand gossiped Ring when used in combination with the previous version of Cortex. Easiest way to upgrade is to shutdown old Cortex installation, and restart it with new version. Incremental rollout works too, but with reduced functionality until all components run the same version. #2016 [FEATURE] Added a read-only local alertmanager config store using files named corresponding to their tenant id. #2125 [FEATURE] Added flag -experimental.ruler.enable-api to enable the ruler api which implements the Prometheus API /api/v1/rules and /api/v1/alerts endpoints under the configured -http.prefix. #1999 [FEATURE] Added sharding support to compactor when using the experimental TSDB blocks storage. #2113 [FEATURE] Added ability to override YAML config file settings using environment variables. #2147 -config.expand-env [FEATURE] Added flags to disable Alertmanager notifications methods. #2187 -configs.notifications.disable-email -configs.notifications.disable-webhook [FEATURE] Add /config HTTP endpoint which exposes the current Cortex configuration as YAML. #2165 [FEATURE] Allow Prometheus remote write directly to ingesters. #1491 [FEATURE] Introduced new standalone service query-tee that can be used for testing purposes to send the same Prometheus query to multiple backends (ie. two Cortex clusters ingesting the same metrics) and compare the performances. #2203 [FEATURE] Fan out parallelizable queries to backend queriers concurrently. #1878 querier.parallelise-shardable-queries (bool) Requires a shard-compatible schema (v10+) This causes the number of traces to increase accordingly. The query-frontend now requires a schema config to determine how/when to shard queries, either from a file or from flags (i.e. by the config-yaml CLI flag). This is the same schema config the queriers consume. The schema is only required to use this option. It’s also advised to increase downstream concurrency controls as well: querier.max-outstanding-requests-per-tenant querier.max-query-parallelism querier.max-concurrent server.grpc-max-concurrent-streams (for both query-frontends and queriers) [FEATURE] Added user sub rings to distribute users to a subset of ingesters. #1947 -experimental.distributor.user-subring-size [FEATURE] Add flag -experimental.tsdb.stripe-size to expose TSDB stripe size option. #2185 [FEATURE] Experimental Delete Series: Added support for Deleting Series with Prometheus style API. Needs to be enabled first by setting -purger.enable to true. Deletion only supported when using boltdb and filesystem as index and object store respectively. Support for other stores to follow in separate PRs #2103 [ENHANCEMENT] Alertmanager: Expose Per-tenant alertmanager metrics #2124 [ENHANCEMENT] Add status label to cortex_alertmanager_configs metric to gauge the number of valid and invalid configs. #2125 [ENHANCEMENT] Cassandra Authentication: added the custom_authenticators config option that allows users to authenticate with cassandra clusters using password authenticators that are not approved by default in gocql #2093 [ENHANCEMENT] Cassandra Storage: added max_retries, retry_min_backoff and retry_max_backoff configuration options to enable retrying recoverable errors. #2054 [ENHANCEMENT] Allow to configure HTTP and gRPC server listen address, maximum number of simultaneous connections and connection keepalive settings. -server.http-listen-address -server.http-conn-limit -server.grpc-listen-address -server.grpc-conn-limit -server.grpc.keepalive.max-connection-idle -server.grpc.keepalive.max-connection-age -server.grpc.keepalive.max-connection-age-grace -server.grpc.keepalive.time -server.grpc.keepalive.timeout [ENHANCEMENT] PostgreSQL: Bump up github.com/lib/pq from v1.0.0 to v1.3.0 to support PostgreSQL SCRAM-SHA-256 authentication. #2097 [ENHANCEMENT] Cassandra Storage: User no longer need CREATE privilege on \u003call keyspaces\u003e if given keyspace exists. #2032 [ENHANCEMENT] Cassandra Storage: added password_file configuration options to enable reading Cassandra password from file. #2096 [ENHANCEMENT] Configs API: Allow GET/POST configs in YAML format. #2181 [ENHANCEMENT] Background cache writes are batched to improve parallelism and observability. #2135 [ENHANCEMENT] Add automatic repair for checkpoint and WAL. #2105 [ENHANCEMENT] Support lastEvaluation and evaluationTime in /api/v1/rules endpoints and make order of groups stable. #2196 [ENHANCEMENT] Skip expired requests in query-frontend scheduling. #2082 [ENHANCEMENT] Add ability to configure gRPC keepalive settings. #2066 [ENHANCEMENT] Experimental TSDB: Export TSDB Syncer metrics from Compactor component, they are prefixed with cortex_compactor_. #2023 [ENHANCEMENT] Experimental TSDB: Added dedicated flag -experimental.tsdb.bucket-store.tenant-sync-concurrency to configure the maximum number of concurrent tenants for which blocks are synched. #2026 [ENHANCEMENT] Experimental TSDB: Expose metrics for objstore operations (prefixed with cortex_\u003ccomponent\u003e_thanos_objstore_, component being one of ingester, querier and compactor). #2027 [ENHANCEMENT] Experimental TSDB: Added support for Azure Storage to be used for block storage, in addition to S3 and GCS. #2083 [ENHANCEMENT] Experimental TSDB: Reduced memory allocations in the ingesters when using the experimental blocks storage. #2057 [ENHANCEMENT] Experimental Memberlist KV: expose -memberlist.gossip-to-dead-nodes-time and -memberlist.dead-node-reclaim-time options to control how memberlist library handles dead nodes and name reuse. #2131 [BUGFIX] Alertmanager: fixed panic upon applying a new config, caused by duplicate metrics registration in the NewPipelineBuilder function. #211 [BUGFIX] Azure Blob ChunkStore: Fixed issue causing invalid chunk checksum errors. #2074 [BUGFIX] The gauge cortex_overrides_last_reload_successful is now only exported by components that use a RuntimeConfigManager. Previously, for components that do not initialize a RuntimeConfigManager (such as the compactor) the gauge was initialized with 0 (indicating error state) and then never updated, resulting in a false-negative permanent error state. #2092 [BUGFIX] Fixed WAL metric names, added the cortex_ prefix. [BUGFIX] Restored histogram cortex_configs_request_duration_seconds #2138 [BUGFIX] Fix wrong syntax for url in config-file-reference. #2148 [BUGFIX] Fixed some 5xx status code returned by the query-frontend when they should actually be 4xx. #2122 [BUGFIX] Fixed leaked goroutines in the querier. #2070 [BUGFIX] Experimental TSDB: fixed /all_user_stats and /api/prom/user_stats endpoints when using the experimental TSDB blocks storage. #2042 [BUGFIX] Experimental TSDB: fixed ruler to correctly work with the experimental TSDB blocks storage. #2101 Changes to denormalised tokens in the ring Cortex 0.4.0 is the last version that can write denormalised tokens. Cortex 0.5.0 and above always write normalised tokens.\nCortex 0.6.0 is the last version that can read denormalised tokens. Starting with Cortex 0.7.0 only normalised tokens are supported, and ingesters writing denormalised tokens to the ring (running Cortex 0.4.0 or earlier with -ingester.normalise-tokens=false) are ignored by distributors. Such ingesters should either switch to using normalised tokens, or be upgraded to Cortex 0.5.0 or later.\nKnown issues The gRPC streaming for ingesters doesn’t work when using the experimental TSDB blocks storage. Please do not enable -querier.ingester-streaming if you’re using the TSDB blocks storage. If you want to enable it, you can build Cortex from master given the issue has been fixed after Cortex 0.7 branch has been cut and the fix wasn’t included in the 0.7 because related to an experimental feature. Annotated config file breaking changes In this section you can find a config file diff showing the breaking changes introduced in Cortex 0.7. You can also find the full configuration file reference doc in the website.\n### Root level config # \"configdb\" has been moved to \"alertmanager \u003e store \u003e configdb\". -[configdb: \u003cconfigdb_config\u003e] # \"config_store\" has been renamed to \"configs\". -[config_store: \u003cconfigstore_config\u003e] +[configs: \u003cconfigs_config\u003e] ### `distributor_config` # The support to hook an external billing system has been removed. -[enable_billing: \u003cboolean\u003e | default = false] -billing: - [maxbufferedevents: \u003cint\u003e | default = 1024] - [retrydelay: \u003cduration\u003e | default = 500ms] - [ingesterhostport: \u003cstring\u003e | default = \"localhost:24225\"] ### `querier_config` # \"ingestermaxquerylookback\" has been renamed to \"query_ingesters_within\". -[ingestermaxquerylookback: \u003cduration\u003e | default = 0s] +[query_ingesters_within: \u003cduration\u003e | default = 0s] ### `queryrange_config` results_cache: cache: # \"defaul_validity\" has been renamed to \"default_validity\". - [defaul_validity: \u003cduration\u003e | default = 0s] + [default_validity: \u003cduration\u003e | default = 0s] # \"cache_split_interval\" has been deprecated in favor of \"split_queries_by_interval\". - [cache_split_interval: \u003cduration\u003e | default = 24h0m0s] ### `alertmanager_config` # The \"store\" config block has been added. This includes \"configdb\" which previously # was the \"configdb\" root level config block. +store: + [type: \u003cstring\u003e | default = \"configdb\"] + [configdb: \u003cconfigstore_config\u003e] + local: + [path: \u003cstring\u003e | default = \"\"] ### `storage_config` index_queries_cache_config: # \"defaul_validity\" has been renamed to \"default_validity\". - [defaul_validity: \u003cduration\u003e | default = 0s] + [default_validity: \u003cduration\u003e | default = 0s] ### `chunk_store_config` chunk_cache_config: # \"defaul_validity\" has been renamed to \"default_validity\". - [defaul_validity: \u003cduration\u003e | default = 0s] + [default_validity: \u003cduration\u003e | default = 0s] write_dedupe_cache_config: # \"defaul_validity\" has been renamed to \"default_validity\". - [defaul_validity: \u003cduration\u003e | default = 0s] + [default_validity: \u003cduration\u003e | default = 0s] # \"min_chunk_age\" has been removed in favor of \"querier \u003e query_store_after\". -[min_chunk_age: \u003cduration\u003e | default = 0s] ### `configs_config` -# \"uri\" has been moved to \"database \u003e uri\". -[uri: \u003cstring\u003e | default = \"postgres://postgres@configs-db.weave.local/configs?sslmode=disable\"] -# \"migrationsdir\" has been moved to \"database \u003e migrations_dir\". -[migrationsdir: \u003cstring\u003e | default = \"\"] -# \"passwordfile\" has been moved to \"database \u003e password_file\". -[passwordfile: \u003cstring\u003e | default = \"\"] +database: + [uri: \u003cstring\u003e | default = \"postgres://postgres@configs-db.weave.local/configs?sslmode=disable\"] + [migrations_dir: \u003cstring\u003e | default = \"\"] + [password_file: \u003cstring\u003e | default = \"\"] 0.6.1 / 2020-02-05 [BUGFIX] Fixed parsing of the WAL configuration when specified in the YAML config file. #2071 0.6.0 / 2020-01-28 Note that the ruler flags need to be changed in this upgrade. You’re moving from a single node ruler to something that might need to be sharded. Further, if you’re using the configs service, we’ve upgraded the migration library and this requires some manual intervention. See full instructions below to upgrade your PostgreSQL.\n[CHANGE] The frontend component now does not cache results if it finds a Cache-Control header and if one of its values is no-store. #1974 [CHANGE] Flags changed with transition to upstream Prometheus rules manager: -ruler.client-timeout is now ruler.configs.client-timeout in order to match ruler.configs.url. -ruler.group-timeouthas been removed. -ruler.num-workers has been removed. -ruler.rule-path has been added to specify where the prometheus rule manager will sync rule files. -ruler.storage.type has beem added to specify the rule store backend type, currently only the configdb. -ruler.poll-interval has been added to specify the interval in which to poll new rule groups. -ruler.evaluation-interval default value has changed from 15s to 1m to match the default evaluation interval in Prometheus. Ruler sharding requires a ring which can be configured via the ring flags prefixed by ruler.ring.. #1987 [CHANGE] Use relative links from /ring page to make it work when used behind reverse proxy. #1896 [CHANGE] Deprecated -distributor.limiter-reload-period flag. #1766 [CHANGE] Ingesters now write only normalised tokens to the ring, although they can still read denormalised tokens used by other ingesters. -ingester.normalise-tokens is now deprecated, and ignored. If you want to switch back to using denormalised tokens, you need to downgrade to Cortex 0.4.0. Previous versions don’t handle claiming tokens from normalised ingesters correctly. #1809 [CHANGE] Overrides mechanism has been renamed to “runtime config”, and is now separate from limits. Runtime config is simply a file that is reloaded by Cortex every couple of seconds. Limits and now also multi KV use this mechanism.\nNew arguments were introduced: -runtime-config.file (defaults to empty) and -runtime-config.reload-period (defaults to 10 seconds), which replace previously used -limits.per-user-override-config and -limits.per-user-override-period options. Old options are still used if -runtime-config.file is not specified. This change is also reflected in YAML configuration, where old limits.per_tenant_override_config and limits.per_tenant_override_period fields are replaced with runtime_config.file and runtime_config.period respectively. #1749 [CHANGE] Cortex now rejects data with duplicate labels. Previously, such data was accepted, with duplicate labels removed with only one value left. #1964 [CHANGE] Changed the default value for -distributor.ha-tracker.prefix from collectors/ to ha-tracker/ in order to not clash with other keys (ie. ring) stored in the same key-value store. #1940 [FEATURE] Experimental: Write-Ahead-Log added in ingesters for more data reliability against ingester crashes. #1103 --ingester.wal-enabled: Setting this to true enables writing to WAL during ingestion. --ingester.wal-dir: Directory where the WAL data should be stored and/or recovered from. --ingester.checkpoint-enabled: Set this to true to enable checkpointing of in-memory chunks to disk. --ingester.checkpoint-duration: This is the interval at which checkpoints should be created. --ingester.recover-from-wal: Set this to true to recover data from an existing WAL. For more information, please checkout the “Ingesters with WAL” guide. [FEATURE] The distributor can now drop labels from samples (similar to the removal of the replica label for HA ingestion) per user via the distributor.drop-label flag. #1726 [FEATURE] Added flag debug.mutex-profile-fraction to enable mutex profiling #1969 [FEATURE] Added global ingestion rate limiter strategy. Deprecated -distributor.limiter-reload-period flag. #1766 [FEATURE] Added support for Microsoft Azure blob storage to be used for storing chunk data. #1913 [FEATURE] Added readiness probe endpoint/ready to queriers. #1934 [FEATURE] Added “multi” KV store that can interact with two other KV stores, primary one for all reads and writes, and secondary one, which only receives writes. Primary/secondary store can be modified in runtime via runtime-config mechanism (previously “overrides”). #1749 [FEATURE] Added support to store ring tokens to a file and read it back on startup, instead of generating/fetching the tokens to/from the ring. This feature can be enabled with the flag -ingester.tokens-file-path. #1750 [FEATURE] Experimental TSDB: Added /series API endpoint support with TSDB blocks storage. #1830 [FEATURE] Experimental TSDB: Added TSDB blocks compactor component, which iterates over users blocks stored in the bucket and compact them according to the configured block ranges. #1942 [ENHANCEMENT] metric cortex_ingester_flush_reasons gets a new reason value: Spread, when -ingester.spread-flushes option is enabled. #1978 [ENHANCEMENT] Added password and enable_tls options to redis cache configuration. Enables usage of Microsoft Azure Cache for Redis service. #1923 [ENHANCEMENT] Upgraded Kubernetes API version for deployments from extensions/v1beta1 to apps/v1. #1941 [ENHANCEMENT] Experimental TSDB: Open existing TSDB on startup to prevent ingester from becoming ready before it can accept writes. The max concurrency is set via --experimental.tsdb.max-tsdb-opening-concurrency-on-startup. #1917 [ENHANCEMENT] Experimental TSDB: Querier now exports aggregate metrics from Thanos bucket store and in memory index cache (many metrics to list, but all have cortex_querier_bucket_store_ or cortex_querier_blocks_index_cache_ prefix). #1996 [ENHANCEMENT] Experimental TSDB: Improved multi-tenant bucket store. #1991 Allowed to configure the blocks sync interval via -experimental.tsdb.bucket-store.sync-interval (0 disables the sync) Limited the number of tenants concurrently synched by -experimental.tsdb.bucket-store.block-sync-concurrency Renamed cortex_querier_sync_seconds metric to cortex_querier_blocks_sync_seconds Track cortex_querier_blocks_sync_seconds metric for the initial sync too [BUGFIX] Fixed unnecessary CAS operations done by the HA tracker when the jitter is enabled. #1861 [BUGFIX] Fixed ingesters getting stuck in a LEAVING state after coming up from an ungraceful exit. #1921 [BUGFIX] Reduce memory usage when ingester Push() errors. #1922 [BUGFIX] Table Manager: Fixed calculation of expected tables and creation of tables from next active schema considering grace period. #1976 [BUGFIX] Experimental TSDB: Fixed ingesters consistency during hand-over when using experimental TSDB blocks storage. #1854 #1818 [BUGFIX] Experimental TSDB: Fixed metrics when using experimental TSDB blocks storage. #1981 #1982 #1990 #1983 [BUGFIX] Experimental memberlist: Use the advertised address when sending packets to other peers of the Gossip memberlist. #1857 [BUGFIX] Experimental TSDB: Fixed incorrect query results introduced in #2604 caused by a buffer incorrectly reused while iterating samples. #2697 Upgrading PostgreSQL (if you’re using configs service) Reference: https://github.com/golang-migrate/migrate/tree/master/database/postgres#upgrading-from-v1\nInstall the migrate package cli tool: https://github.com/golang-migrate/migrate/tree/master/cmd/migrate#installation Drop the schema_migrations table: DROP TABLE schema_migrations;. Run the migrate command: migrate -path \u003cabsolute_path_to_cortex\u003e/cmd/cortex/migrations -database postgres://localhost:5432/database force 2 Known issues The cortex_prometheus_rule_group_last_evaluation_timestamp_seconds metric, tracked by the ruler, is not unregistered for rule groups not being used anymore. This issue will be fixed in the next Cortex release (see 2033).\nWrite-Ahead-Log (WAL) does not have automatic repair of corrupt checkpoint or WAL segments, which is possible if ingester crashes abruptly or the underlying disk corrupts. Currently the only way to resolve this is to manually delete the affected checkpoint and/or WAL segments. Automatic repair will be added in the future releases.\n0.4.0 / 2019-12-02 [CHANGE] The frontend component has been refactored to be easier to reuse. When upgrading the frontend, cache entries will be discarded and re-created with the new protobuf schema. #1734 [CHANGE] Removed direct DB/API access from the ruler. -ruler.configs.url has been now deprecated. #1579 [CHANGE] Removed Delta encoding. Any old chunks with Delta encoding cannot be read anymore. If ingester.chunk-encoding is set to Delta the ingester will fail to start. #1706 [CHANGE] Setting -ingester.max-transfer-retries to 0 now disables hand-over when ingester is shutting down. Previously, zero meant infinite number of attempts. #1771 [CHANGE] dynamo has been removed as a valid storage name to make it consistent for all components. aws and aws-dynamo remain as valid storage names. [CHANGE/FEATURE] The frontend split and cache intervals can now be configured using the respective flag --querier.split-queries-by-interval and --frontend.cache-split-interval. If --querier.split-queries-by-interval is not provided request splitting is disabled by default. --querier.split-queries-by-day is still accepted for backward compatibility but has been deprecated. You should now use --querier.split-queries-by-interval. We recommend a to use a multiple of 24 hours. [FEATURE] Global limit on the max series per user and metric #1760 -ingester.max-global-series-per-user -ingester.max-global-series-per-metric Requires -distributor.replication-factor and -distributor.shard-by-all-labels set for the ingesters too [FEATURE] Flush chunks with stale markers early with ingester.max-stale-chunk-idle. #1759 [FEATURE] EXPERIMENTAL: Added new KV Store backend based on memberlist library. Components can gossip about tokens and ingester states, instead of using Consul or Etcd. #1721 [FEATURE] EXPERIMENTAL: Use TSDB in the ingesters \u0026 flush blocks to S3/GCS ala Thanos. This will let us use an Object Store more efficiently and reduce costs. #1695 [FEATURE] Allow Query Frontend to log slow queries with frontend.log-queries-longer-than. #1744 [FEATURE] Add HTTP handler to trigger ingester flush \u0026 shutdown - used when running as a stateful set with the WAL enabled. #1746 [FEATURE] EXPERIMENTAL: Added GCS support to TSDB blocks storage. #1772 [ENHANCEMENT] Reduce memory allocations in the write path. #1706 [ENHANCEMENT] Consul client now follows recommended practices for blocking queries wrt returned Index value. #1708 [ENHANCEMENT] Consul client can optionally rate-limit itself during Watch (used e.g. by ring watchers) and WatchPrefix (used by HA feature) operations. Rate limiting is disabled by default. New flags added: --consul.watch-rate-limit, and --consul.watch-burst-size. #1708 [ENHANCEMENT] Added jitter to HA deduping heartbeats, configure using distributor.ha-tracker.update-timeout-jitter-max #1534 [ENHANCEMENT] Add ability to flush chunks with stale markers early. #1759 [BUGFIX] Stop reporting successful actions as 500 errors in KV store metrics. #1798 [BUGFIX] Fix bug where duplicate labels can be returned through metadata APIs. #1790 [BUGFIX] Fix reading of old, v3 chunk data. #1779 [BUGFIX] Now support IAM roles in service accounts in AWS EKS. #1803 [BUGFIX] Fixed duplicated series returned when querying both ingesters and store with the experimental TSDB blocks storage. #1778 In this release we updated the following dependencies:\ngRPC v1.25.0 (resulted in a drop of 30% CPU usage when compression is on) jaeger-client v2.20.0 aws-sdk-go to v1.25.22 0.3.0 / 2019-10-11 This release adds support for Redis as an alternative to Memcached, and also includes many optimisations which reduce CPU and memory usage.\n[CHANGE] Gauge metrics were renamed to drop the _total suffix. #1685 In Alertmanager, alertmanager_configs_total is now alertmanager_configs In Ruler, scheduler_configs_total is now scheduler_configs scheduler_groups_total is now scheduler_groups. [CHANGE] --alertmanager.configs.auto-slack-root flag was dropped as auto Slack root is not supported anymore. #1597 [CHANGE] In table-manager, default DynamoDB capacity was reduced from 3,000 units to 1,000 units. We recommend you do not run with the defaults: find out what figures are needed for your environment and set that via -dynamodb.periodic-table.write-throughput and -dynamodb.chunk-table.write-throughput. [FEATURE] Add Redis support for caching #1612 [FEATURE] Allow spreading chunk writes across multiple S3 buckets #1625 [FEATURE] Added /shutdown endpoint for ingester to shutdown all operations of the ingester. #1746 [ENHANCEMENT] Upgraded Prometheus to 2.12.0 and Alertmanager to 0.19.0. #1597 [ENHANCEMENT] Cortex is now built with Go 1.13 #1675, #1676, #1679 [ENHANCEMENT] Many optimisations, mostly impacting ingester and querier: #1574, #1624, #1638, #1644, #1649, #1654, #1702 Full list of changes: https://github.com/cortexproject/cortex/compare/v0.2.0...v0.3.0\n0.2.0 / 2019-09-05 This release has several exciting features, the most notable of them being setting -ingester.spread-flushes to potentially reduce your storage space by upto 50%.\n[CHANGE] Flags changed due to changes upstream in Prometheus Alertmanager #929: alertmanager.mesh.listen-address is now cluster.listen-address alertmanager.mesh.peer.host and alertmanager.mesh.peer.service can be replaced by cluster.peer alertmanager.mesh.hardware-address, alertmanager.mesh.nickname, alertmanager.mesh.password, and alertmanager.mesh.peer.refresh-interval all disappear. [CHANGE] –claim-on-rollout flag deprecated; feature is now always on #1566 [CHANGE] Retention period must now be a multiple of periodic table duration #1564 [CHANGE] The value for the name label for the chunks memcache in all cortex_cache_ metrics is now chunksmemcache (before it was memcache) #1569 [FEATURE] Makes the ingester flush each timeseries at a specific point in the max-chunk-age cycle with -ingester.spread-flushes. This means multiple replicas of a chunk are very likely to contain the same contents which cuts chunk storage space by up to 66%. #1578 [FEATURE] Make minimum number of chunk samples configurable per user #1620 [FEATURE] Honor HTTPS for custom S3 URLs #1603 [FEATURE] You can now point the query-frontend at a normal Prometheus for parallelisation and caching #1441 [FEATURE] You can now specify http_config on alert receivers #929 [FEATURE] Add option to use jump hashing to load balance requests to memcached #1554 [FEATURE] Add status page for HA tracker to distributors #1546 [FEATURE] The distributor ring page is now easier to read with alternate rows grayed out #1621 0.1.0 / 2019-08-07 [CHANGE] HA Tracker flags were renamed to provide more clarity #1465 distributor.accept-ha-labels is now distributor.ha-tracker.enable distributor.accept-ha-samples is now distributor.ha-tracker.enable-for-all-users ha-tracker.replica is now distributor.ha-tracker.replica ha-tracker.cluster is now distributor.ha-tracker.cluster [FEATURE] You can specify “heap ballast” to reduce Go GC Churn #1489 [BUGFIX] HA Tracker no longer always makes a request to Consul/Etcd when a request is not from the active replica #1516 [BUGFIX] Queries are now correctly cancelled by the query-frontend #1508 ","categories":"","description":"","excerpt":"master / unreleased [CHANGE] OTLP: Change OTLP handler to be …","ref":"/docs/changelog/","tags":"","title":"Changelog"},{"body":"Cortex follows the CNCF Code of Conduct.\n","categories":"","description":"","excerpt":"Cortex follows the CNCF Code of Conduct.\n","ref":"/docs/code-of-conduct/","tags":"","title":"Code of Conduct"},{"body":"In this page we’re listing some external resources to learn more about the Cortex blocks storage.\nTalks Scaling Prometheus: How We Got Some Thanos Into Cortex Articles How to switch Cortex from chunks to blocks storage (and why you won’t look back) (Oct 2020) Now GA: Cortex blocks storage for running Prometheus at scale with reduced operational complexity (Oct 2020) Scaling Prometheus: How we’re pushing Cortex blocks storage to its limit and beyond (Aug 2020) How blocks storage in Cortex reduces operational complexity for running Prometheus at massive scale (Jul 2020) ","categories":"","description":"","excerpt":"In this page we’re listing some external resources to learn more about …","ref":"/docs/blocks-storage/learn-more/","tags":"","title":"Learn more"},{"body":"Tenant ID naming The tenant ID (also called “user ID” or “org ID”) is the unique identifier of a tenant within a Cortex cluster. The tenant ID is opaque information to Cortex, which doesn’t make any assumptions on its format/content, but its naming has two limitations:\nSupported characters Length Supported characters The following character sets are generally safe for use in the tenant ID:\nAlphanumeric characters 0-9 a-z A-Z Special characters Exclamation point (!) Hyphen (-) Underscore (_) Single Period (.), but the tenant IDs . and .. are considered invalid Asterisk (*) Single quote (') Open parenthesis (() Close parenthesis ()) All other characters are not safe to use. In particular, slashes / and whitespaces ( ) are not supported.\nLength The tenant ID length should not exceed 150 bytes/characters.\n","categories":"","description":"","excerpt":"Tenant ID naming The tenant ID (also called “user ID” or “org ID”) is …","ref":"/docs/guides/limitations/","tags":"","title":"Limitations"},{"body":"Blocks storage The blocks storage is a Cortex storage engine based on Prometheus TSDB, which only requires an object store (eg. AWS S3, Google GCS, …) as backend storage.\nFor more information, please refer to the Cortex blocks storage documentation.\nChunk A chunk is an object containing compressed timestamp-value pairs.\nA single chunk contains timestamp-value pairs for several series.\nChurn Churn is the frequency at which series become idle.\nA series becomes idle once it’s not exported anymore by the monitored targets. Typically, series become idle when the monitored target itself disappears (eg. the process or node gets terminated).\nFlushing Series flushing is the operation run by ingesters to offload time series from memory and store them in the long-term storage.\nHA Tracker The HA Tracker is a feature of Cortex distributor which is used to deduplicate received series coming from two (or more) Prometheus servers configured in HA pairs.\nFor more information, please refer to the guide “Config for sending HA Pairs data to Cortex”.\nHash ring The hash ring is a distributed data structure used by Cortex for sharding, replication, and service discovery. The hash ring data structure gets shared across Cortex replicas via gossip or a key-value store.\nFor more information, please refer to the Architecture documentation.\nOrg See Tenant.\nRing See Hash ring.\nSample A sample is a single timestamped value in a time series.\nFor example, given the series node_cpu_seconds_total{instance=\"10.0.0.1\",mode=\"system\"} its stream of values (samples) could be:\n# Display format: \u003cvalue\u003e @\u003ctimestamp\u003e 11775 @1603812134 11790 @1603812149 11805 @1603812164 11819 @1603812179 11834 @1603812194 Series In the Prometheus ecosystem, a series (or time series) is a single stream of timestamped values belonging to the same metric, with the same set of label key-value pairs.\nFor example, given a single metric node_cpu_seconds_total you may have multiple series, each one uniquely identified by the combination of metric name and unique label key-value pairs:\nnode_cpu_seconds_total{instance=\"10.0.0.1\",mode=\"system\"} node_cpu_seconds_total{instance=\"10.0.0.1\",mode=\"user\"} node_cpu_seconds_total{instance=\"10.0.0.2\",mode=\"system\"} node_cpu_seconds_total{instance=\"10.0.0.2\",mode=\"user\"} Tenant A tenant (also called “user” or “org”) is the owner of a set of series written to and queried from Cortex. Cortex multi-tenancy support allows you to isolate series belonging to different tenants. For example, if you have two tenants team-A and team-B, team-A series will be isolated from team-B, and each team will be able to query only their own series.\nFor more information, please refer to:\nHTTP API authentication Tenant ID limitations Time series See Series.\nUser See Tenant.\nWAL The Write-Ahead Log (WAL) is an append-only log stored on disk used by ingesters to recover their in-memory state after the process gets restarted, either after a clear shutdown or an abrupt termination.\nFor more information, please refer to Ingesters with WAL.\n","categories":"","description":"","excerpt":"Blocks storage The blocks storage is a Cortex storage engine based on …","ref":"/docs/guides/glossary/","tags":"","title":"Glossary"},{"body":"","categories":"","description":"","excerpt":"","ref":"/index.json","tags":"","title":""},{"body":" Horizontally scalable, highly available, multi-tenant, long term storage for Prometheus. Learn More Releases Companies using Cortex\nLong term storage Durably store data for longer than the lifetime of any single machine, and use this data for long term capacity planning. Blazin’ fast PromQL Cortex makes your PromQL queries blazin' fast through aggressive parallelization and caching. A global view of data Cortex gives you a global view of Prometheus time series data that includes data in long-term storage, greatly expanding the usefulness of PromQL for analytical purposes. Horizontally scalable Cortex runs across multiple machines in a cluster, exceeding the throughput and storage of a single machine. This enables you to send the metrics from multiple Prometheus servers to a single Cortex cluster. We are a Cloud Native Computing Foundation Incubating project.\nJoin the community ! Join users and companies that are using Cortex in production.\nSlack Issues Twitter ","categories":"","description":"","excerpt":" Horizontally scalable, highly available, multi-tenant, long term …","ref":"/","tags":"","title":"Cortex"},{"body":"","categories":"","description":"","excerpt":"","ref":"/search/","tags":"","title":"Search Results"}]
\ No newline at end of file
diff --git a/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json b/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json
new file mode 100644
index 0000000000..600a51f144
--- /dev/null
+++ b/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json
@@ -0,0 +1 @@
+[{"body":" Author: Doğukan Teber Date: March 2023 Status: Proposed Overview If you run Cortex for multiple tenants you need to identify your tenants every time they send metrics or query them. This is needed to ensure that metrics can be ingested and queried separately from each other. For this purpose, the Cortex microservices require you to pass a Header called X-Scope-OrgID. The Cortex k8s manifests suggest deploying an NGINX server inside of each tenant which acts as a reverse proxy. Its sole purpose is proxying the traffic and setting the X-Scope-OrgID header for your tenant.\nProposal We propose to solve this problem by adding a gateway that can be considered the entry point for all requests towards Cortex. This gateway will be responsible for handling multi-tenancy features without complicated proxy configuration.\nThis gateway will be located in another repository, cortexproject/auth-gateway. For the first version, basic authentication will be supported. However, as the project grows, it is planned to support different authentication mechanisms according to the needs of the users.\nWe would like to have a simple configuration format. The following is the proposed configuration:\nserver: address: localhost port: 8080 debug: address: localhost port: 8090 tenants: - authentication: basic username: username1 password: password1 id: orgid - authentication: basic username: username2 password: password2 id: orgid distributor: dns_refresh_interval: 3s url: http://distributor:9009 paths: - /api/v1/push - /api/prom/push http_client_timeout: 20 read_timeout: 5 write_timeout: 5 idle_timeout: 7 frontend: url: http://frontend:9010 read_timeout: 5 write_timeout: 5 idle_timeout: 7 ... debug will be used for endpoints that do not require authentication, ie. /metrics, /ready, /pprof. If no paths are given in the component, such as frontend, the default API paths of that component will be registered. If paths are provided, then the provided API paths will override the default API paths. Additionally, the users will be able to specify custom timeouts for each component similar to NGINX.\nNote that only the distributor and query-frontend are mentioned in the example configuration. However, in addition to the distributor and query-frontend, alertmanager and ruler will be supported. The same configurations can be done for these components as well.\nThe time values that are specified in the above configuration are not tested and are only given as an example. The default values for these fields will be decided later when comprehensive tests are conducted.\nFinally, we will ensure the load is balanced between the targets similar to NGINX in which there is a configurable dns refresh parameter and a round-robin between all the hosts.\n","categories":"","description":"","excerpt":" Author: Doğukan Teber Date: March 2023 Status: Proposed Overview If …","ref":"/docs/proposals/auth-gateway/","tags":"","title":"Authentication Gateway"},{"body":" Author: Marco Pracucci Date: November 2020 Status: draft Introduction Queriers and store-gateways, at startup and periodically while running, need to scan the entire object store bucket to find tenants and blocks for each tenants.\nFor each discovered block, they need to fetch the meta.json file and check for the existence of deletion-mark.json (used to signal a block is “marked for deletion”). The meta.json file is immutable and gets cached, but deletion-mark.json can be created at any time and its non-existence can’t be safely cached for long (actually our cache is pretty ineffective regarding this).\nThe number of blocks in the storage linearly increases with the number of tenants, and so does the number of bucket API calls required to scan the bucket.\nFor example, assuming 1 block / day / tenant and a 400d retention, we would have 400 blocks for a single-tenant cluster, but 4M blocks for a 10K tenants cluster (regardless of the number of active series or their QPS).\nWe’re currently testing a cluster running with 6K tenants and we have faced the following issues:\nThe cold start of queriers and store-gateways (empty caches) take tens of minutes. If we increase the concurrency of bucket operations run to scan the bucket, we hit into object store rate limits and/or the 5xx error rate increases The need to run “list objects” API call for queriers and store-gateways to discover blocks and the check for the existence of deletion-mark.json (which requires a read operation for each block and can’t be cached for a long time) represents a significant % of bucket API calls baseline costs, regardless the tenants QPS (costs we have even if the cluster has 0 queries) Goal The goal of this proposal is:\nThe querier should be up and running without having to scan the bucket at all (zero startup time) The querier should not run any “list objects” operation at anytime The querier should require only 1 “get object” operation to get the entire view of a tenant’s blocks Out of scope We believe the same technique described in this proposal could be applied to optimise the store-gateway too (we’ve built a PoC), however to keep the design doc easier to discuss we suggest to keep the store-gateway out of scope and address it in a follow-up design doc.\nProposal We propose to introduce a per-tenant bucket index. The index is a single JSON file containing two main information: list of all completed (non partial) blocks in the bucket + list of all deletion marks. The bucket index is stored in the bucket within the tenant location (eg. /user-1/bucket-index.json) and is kept updated by the compactor.\nThe querier, at query time, checks whether the bucket index for the tenant has already been loaded in memory. If not, the querier will download it and cache it in memory. Given it’s a small file, we expect the lazy download of the bucket index to not significantly impact first query performances.\nWhile in-memory, a background process will keep it updated at periodic intervals (configurable), so that subsequent queries from the same tenant to the same querier instance will use the cached (and periodically updated) bucket index.\nIf a bucket index is unused for a long time (configurable), eg. because that querier instance is not receiving any query from the tenant, the querier will offload it, stopping to keep it updated at regular intervals. This is particularly useful when shuffle sharding is enabled, because a querier will only run queries for a subset of tenants and tenants can be re-sharded across queries in the event of a scale up/down or rolling updates.\nThe bucket index will be also cached on memcached for a short period of time, to reduce the number of “get object” operations in case multiple queriers fetch it in a short period of time (eg. 5 minutes).\nBucket index structure The bucket-index.json is a single file containing the following information:\nVersion List of blocks ID, MinTime, MaxTime, UploadedAt SegmentsFormat (eg. “1-based-6-digits”), SegmentsNum (these two information will be used by the store-gateway) List of block deletion marks ID, DeletionTime Timestamp of when the index has been updated The bucket index intentionally stores a subset of data for each block’s meta.json, in order to keep the size of the index small. The information in the index is enough to run the querier without having to load the full meta.json of each block.\nSize: the expected size of the index is about 150 bytes per block. In the case of a tenant with 400 blocks, its bucket index would be 58KB. The size could be further reduced compressing it: experiments show a compression ratio of about 4x using gzip.\nWhy the compactor writes the bucket index There are few reasons why the compactor may be a good candidate to write the bucket index:\nThe compactor already scans the bucket to have a consistent view of the blocks before running a compaction. Writing the bucket index in the compactor would require no additional bucket API calls. Queriers and store-gateways currently read only from the bucket (no writes). We believe that it may be a nice property to preserve. How to reduce bucket API calls required to discover deletion marks The deletion-mark.json files are stored within each block location. This means that the compactor would still need to run a “get object” operation for every single block in order to find out which block has it, while updating the bucket index.\nTo reduce the number of operations required, we propose to store block deletion marks for all blocks in a per-tenant location (markers/). Discovering all blocks marked for deletion for a given tenant would only require a single “list objects” operation on the /\u003ctenant-id\u003e/markers/ prefix. New markers introduced in the future (eg. tenant deletion mark) could be stored in the same location in order to discover all markers with a single “list objects” operation.\nFor example:\n/user-1/markers/01ER1ZSYHF1FT6RBD8HTVQWX13-deletion-mark.json /user-1/markers/01ER1ZGSX1Q4B41MK1QQ7RHD33-deletion-mark.json /user-1/markers/01ER1ZGWKVFT60YMXT8D3XJMDB-deletion-mark.json What if a bucket index gets stale Queriers and store-gateways don’t need a strongly consistent view over the bucket. Even today, given queriers and store-gateways run a periodic scan of the bucket, their view could be stale up to the “scan interval” (defaults to 15m).\nThe maximum delay between a change into the bucket is picked up by queriers and store-gateways depends on the configuration and is the minimum time between:\nNew blocks uploaded by ingester: min(-querier.query-ingesters-within, -blocks-storage.tsdb.retention-period) (default: 6h) New blocks uploaded/deleted by compactor: -compactor.deletion-delay (default: 12h) In order to guarantee consistent query results, we propose to configure the bucket index max stale period in the querier and fail queries if, because of any issue, the bucket index UpdatedAt age exceeds it.\nThe staleness of each bucket index will be tracked via metrics, in order to be alert on it when it’s close to expiration (but before it happens).\nObject store eventual consistency Object stores like S3 or GCS do guarantee read-after-create consistency but not read-after-update.\nHowever, given the analysis done in “What if a bucket index gets stale” we expect this not to be a real issue, considering we do expect object store reads to not be out of sync for hours.\n","categories":"","description":"","excerpt":" Author: Marco Pracucci Date: November 2020 Status: draft Introduction …","ref":"/docs/proposals/blocks-storage-bucket-index/","tags":"","title":"Blocks storage bucket index"},{"body":" Author: Marco Pracucci Date: March 2020 Status: accepted Problem In Cortex, when using the experimental blocks storage, each querier internally runs the Thanos BucketStore. This means that each querier has a full view over all blocks in the long-term storage and all blocks index headers are loaded in each querier memory. The querier memory usage linearly increase with number and size of all blocks in the storage, imposing a scalability limit to the blocks storage.\nIn this proposal we want to solve this. In particular, we want to:\nShard blocks (index headers) across a pool of nodes Do not compromise HA on the read path (if a node fails, queries should continue to work) Do not compromise correctness (either the query result is correct or it fails) Proposed solution The idea is to introduce a new Cortex service - store-gateway - internally running the Thanos BucketStore. At query time, a querier will run a query fetching the matching series both from ingesters and the subset of gateways holding the related blocks (based on the query time range). Blocks are replicated across the gateways in order to guarantee query results consistency and HA even in the case of a gateway instance failure.\nRing-based sharding and replication In order to build blocks sharding and replication, the store-gateway instances form a ring. Each gateway instance uses a custom MetaFetcherFilter to filter blocks loaded on the instance itself, keeping only blocks whose hash(block-id) is within the tokens range assigned to the gateway instance within the ring.\nWithin a gateway, the blocks synchronization is triggered in two cases:\nPeriodically\nto discover new blocks uploaded by ingesters or compactor, and delete old blocks removed due to retention or by the compactor On-demand when the ring topology changes (the tokens ranges assigned to the gateway instance have changed) It’s important to outline that the sync takes time (typically will have to re-scan the bucket and download new blocks index headers) and Cortex needs to guarantee query results consistency at any given time (see below).\nQuery execution When a querier executes a query, it will need to fetch series both from ingesters and the store-gateway instances.\nFor a given query, the number of blocks to query is expected to be low, especially if the Cortex cluster is running the query-frontend with a 24h query split interval. In this scenario, whatever is the client’s query time range, the query-frontend will split the client’s query into partitioned queries each with up to 24h time range and the querier will likely hit not more than 1 block per partitioned query (except for the last 24h for which blocks may have not been compacted yet).\nGiven this assumption, we want to avoid sending every query to every store-gateway instance. The querier should be able to take an informed decision about the minimum subset of store-gateway instances which needs to query given a time range.\nThe idea is to run the MetaFetcher also within the querier, but without any sharding filter (contrary to the store-gateway). At any given point in time, the querier knows the entire list of blocks in the storage. When the querier executes the Select() (or SelectSorted()) it does:\nCompute the list of blocks by the query time range Compute the minimum list of store-gateway instances containing the required blocks (using the information from the ring) Fetch series from ingesters and the matching store-gateway instances Merge and deduplicate received series Optimization: can be skipped if the querier hits only 1 store-gateway Query results consistency When a querier executes a query, it should guarantee that either all blocks matching the time range are queried or the query fails.\nHowever, due to the (intentional) lack of a strong coordination between queriers and store-gateways, and the ring topology which can change any time, there’s no guarantee that the blocks assigned to a store-gateway shard are effectively loaded on the store-gateway itself at any given point in time.\nThe idea is introduce a consistency check in the querier. When a store-gateway receives a request from the querier, the store-gateway includes in the response the list of block IDs currently loaded on the store-gateway itself. The querier can then merge the list of block IDs received from all store-gateway hit, and match it against the list of block IDs computed at the beginning of the query execution.\nThere are three possible scenarios:\nThe list match: all good All the blocks known by the querier are within the list of blocks returned by store-gateway, but the store-gateway also included blocks unknown to the querier: all good (it means the store-gateways have discovered and loaded new blocks before the querier discovered them) Some blocks known by the querier are not within the list of blocks returned by store-gateway: potential consistency issue We want to protect from a partial results response which may occur in the case #3. However, there are some legit cases which, if not handled, would lead to frequent false positives. Given the querier and store-gateway instances independently scan the bucket at a regular interval (to find new blocks or deleted blocks), we may be in one of the following cases:\na. The querier has discovered new blocks before the store-gateway successfully discovered and loaded them b. The store-gateway has offloaded blocks “marked for deletion” before the querier\nTo protect from case (a), we can exclude the blocks which have been uploaded in the last X time from the consistency check (same technique already used in other Thanos components). This X delay time is used to give the store-gateway enough time to discover and load new blocks, before the querier consider them for the consistency check. This value X should be greater than the -experimental.blocks-storage.bucket-store.consistency-delay, because we do expect the querier to consider a block for consistency check once it’s reasonably safe to assume that its store-gateway already loaded it.\nTo protect from case (b) we need to understand how blocks are offloaded. The BucketStore (running within the store-gateway) offloads a block as soon as it’s not returned by the MetaFetcher. This means we can configure the MetaFetcher with a IgnoreDeletionMarkFilter with a delay of X (could be the same value used for case (a)) and in the querier exclude the blocks which have been marked for deletion more than X time ago from the consistency check.\nTrade-offs The proposed solution comes with the following trade-offs:\nA querier is not ready until it has completed an initial full scan of the bucket, downloading the meta.json file of every block A store-gateway is not ready until it has completed an initial full scan of the bucket, downloading the meta.json and index header of each block matching its shard If a querier hits 2+ store-gateways it may receive duplicated series if the 2+ store-gateways share some blocks due to the replication factor ","categories":"","description":"","excerpt":" Author: Marco Pracucci Date: March 2020 Status: accepted Problem In …","ref":"/docs/proposals/blocks-storage-sharding/","tags":"","title":"Blocks storage sharding"},{"body":" Cortex can be configured using a YAML file - specified using the -config.file flag - or CLI flags. In case you combine both, CLI flags take precedence over the YAML config file.\nThe current configuration of any Cortex component can be seen by visiting the /config HTTP path. Passwords are filtered out of this endpoint.\nReference To specify which configuration file to load, pass the -config.file flag at the command line. The file is written in YAML format, defined by the scheme below. Brackets indicate that a parameter is optional.\nGeneric placeholders \u003cboolean\u003e: a boolean that can take the values true or false \u003cint\u003e: any integer matching the regular expression [1-9]+[0-9]* \u003cduration\u003e: a duration matching the regular expression [0-9]+(ns|us|µs|ms|s|m|h|d|w|y) where y = 365 days. \u003cstring\u003e: a regular string \u003curl\u003e: a URL \u003cprefix\u003e: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used) \u003crelabel_config\u003e: a Prometheus relabeling configuration. \u003ctime\u003e: a timestamp, with available formats: 2006-01-20 (midnight, local timezone), 2006-01-20T15:04 (local timezone), and RFC 3339 formats: 2006-01-20T15:04:05Z (UTC) or 2006-01-20T15:04:05+07:00 (explicit timezone) Use environment variables in the configuration You can use environment variable references in the config file to set values that need to be configurable during deployment by using the -config.expand-env flag. To do this, use:\n${VAR} Where VAR is the name of the environment variable.\nEach variable reference is replaced at startup by the value of the environment variable. The replacement is case-sensitive and occurs before the YAML file is parsed. References to undefined variables are replaced by empty strings unless you specify a default value or custom error text.\nTo specify a default value, use:\n${VAR:default_value} Where default_value is the value to use if the environment variable is undefined.\nSupported contents and default values of the config file # Comma-separated list of Cortex modules to load. The alias 'all' can be used in # the list to load a number of core modules and will enable single-binary mode. # Use '-modules' command line flag to get a list of available modules, and to # see which modules are included in 'all'. # CLI flag: -target [target: \u003cstring\u003e | default = \"all\"] # Set to false to disable auth. # CLI flag: -auth.enabled [auth_enabled: \u003cboolean\u003e | default = true] # HTTP path prefix for Cortex API. # CLI flag: -http.prefix [http_prefix: \u003cstring\u003e | default = \"/api/prom\"] api: # Use GZIP compression for API responses. Some endpoints serve large YAML or # JSON blobs which can benefit from compression. # CLI flag: -api.response-compression-enabled [response_compression_enabled: \u003cboolean\u003e | default = false] # HTTP URL path under which the Alertmanager ui and api will be served. # CLI flag: -http.alertmanager-http-prefix [alertmanager_http_prefix: \u003cstring\u003e | default = \"/alertmanager\"] # HTTP URL path under which the Prometheus api will be served. # CLI flag: -http.prometheus-http-prefix [prometheus_http_prefix: \u003cstring\u003e | default = \"/prometheus\"] # Which HTTP Request headers to add to logs # CLI flag: -api.http-request-headers-to-log [http_request_headers_to_log: \u003clist of string\u003e | default = []] # Regex for CORS origin. It is fully anchored. Example: # 'https?://(domain1|domain2)\\.com' # CLI flag: -server.cors-origin [cors_origin: \u003cstring\u003e | default = \".*\"] # If enabled, build Info API will be served by query frontend or querier. # CLI flag: -api.build-info-enabled [build_info_enabled: \u003cboolean\u003e | default = false] # Choose default codec for querier response serialization. Supports 'json' and # 'protobuf'. # CLI flag: -api.querier-default-codec [querier_default_codec: \u003cstring\u003e | default = \"json\"] # The server_config configures the HTTP and gRPC server of the launched # service(s). [server: \u003cserver_config\u003e] # The distributor_config configures the Cortex distributor. [distributor: \u003cdistributor_config\u003e] # The querier_config configures the Cortex querier. [querier: \u003cquerier_config\u003e] # The ingester_client_config configures how the Cortex distributors connect to # the ingesters. [ingester_client: \u003cingester_client_config\u003e] # The ingester_config configures the Cortex ingester. [ingester: \u003cingester_config\u003e] # The flusher_config configures the WAL flusher target, used to manually run # one-time flushes when scaling down ingesters. [flusher: \u003cflusher_config\u003e] # The storage_config configures the storage type Cortex uses. [storage: \u003cstorage_config\u003e] # The limits_config configures default and per-tenant limits imposed by Cortex # services (ie. distributor, ingester, ...). [limits: \u003climits_config\u003e] # The frontend_worker_config configures the worker - running within the Cortex # querier - picking up and executing queries enqueued by the query-frontend or # query-scheduler. [frontend_worker: \u003cfrontend_worker_config\u003e] # The query_frontend_config configures the Cortex query-frontend. [frontend: \u003cquery_frontend_config\u003e] # The query_range_config configures the query splitting and caching in the # Cortex query-frontend. [query_range: \u003cquery_range_config\u003e] # The blocks_storage_config configures the blocks storage. [blocks_storage: \u003cblocks_storage_config\u003e] # The compactor_config configures the compactor for the blocks storage. [compactor: \u003ccompactor_config\u003e] # The store_gateway_config configures the store-gateway service used by the # blocks storage. [store_gateway: \u003cstore_gateway_config\u003e] tenant_federation: # If enabled on all Cortex services, queries can be federated across multiple # tenants. The tenant IDs involved need to be specified separated by a `|` # character in the `X-Scope-OrgID` header (experimental). # CLI flag: -tenant-federation.enabled [enabled: \u003cboolean\u003e | default = false] # The ruler_config configures the Cortex ruler. [ruler: \u003cruler_config\u003e] # The ruler_storage_config configures the Cortex ruler storage backend. [ruler_storage: \u003cruler_storage_config\u003e] # The configs_config configures the Cortex Configs DB and API. [configs: \u003cconfigs_config\u003e] # The alertmanager_config configures the Cortex alertmanager. [alertmanager: \u003calertmanager_config\u003e] # The alertmanager_storage_config configures the Cortex alertmanager storage # backend. [alertmanager_storage: \u003calertmanager_storage_config\u003e] # The runtime_configuration_storage_config configures the storage backend for # the runtime configuration file. [runtime_config: \u003cruntime_configuration_storage_config\u003e] # The memberlist_config configures the Gossip memberlist. [memberlist: \u003cmemberlist_config\u003e] query_scheduler: # If a querier disconnects without sending notification about graceful # shutdown, the query-scheduler will keep the querier in the tenant's shard # until the forget delay has passed. This feature is useful to reduce the # blast radius when shuffle-sharding is enabled. # CLI flag: -query-scheduler.querier-forget-delay [querier_forget_delay: \u003cduration\u003e | default = 0s] # This configures the gRPC client used to report errors back to the # query-frontend. grpc_client_config: # gRPC client max receive message size (bytes). # CLI flag: -query-scheduler.grpc-client-config.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # gRPC client max send message size (bytes). # CLI flag: -query-scheduler.grpc-client-config.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 16777216] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy', 'snappy-block' ,'zstd' and '' (disable compression) # CLI flag: -query-scheduler.grpc-client-config.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # Rate limit for gRPC client; 0 means disabled. # CLI flag: -query-scheduler.grpc-client-config.grpc-client-rate-limit [rate_limit: \u003cfloat\u003e | default = 0] # Rate limit burst for gRPC client. # CLI flag: -query-scheduler.grpc-client-config.grpc-client-rate-limit-burst [rate_limit_burst: \u003cint\u003e | default = 0] # Enable backoff and retry when we hit ratelimits. # CLI flag: -query-scheduler.grpc-client-config.backoff-on-ratelimits [backoff_on_ratelimits: \u003cboolean\u003e | default = false] backoff_config: # Minimum delay when backing off. # CLI flag: -query-scheduler.grpc-client-config.backoff-min-period [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -query-scheduler.grpc-client-config.backoff-max-period [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -query-scheduler.grpc-client-config.backoff-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS in the GRPC client. This flag needs to be enabled when any # other TLS flag is set. If set to false, insecure connection to gRPC server # will be used. # CLI flag: -query-scheduler.grpc-client-config.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -query-scheduler.grpc-client-config.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -query-scheduler.grpc-client-config.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. # If not set, the host's root CA certificates are used. # CLI flag: -query-scheduler.grpc-client-config.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -query-scheduler.grpc-client-config.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -query-scheduler.grpc-client-config.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # The tracing_config configures backends cortex uses. [tracing: \u003ctracing_config\u003e] alertmanager_config The alertmanager_config configures the Cortex alertmanager.\n# Base path for data storage. # CLI flag: -alertmanager.storage.path [data_dir: \u003cstring\u003e | default = \"data/\"] # How long to keep data for. # CLI flag: -alertmanager.storage.retention [retention: \u003cduration\u003e | default = 120h] # The URL under which Alertmanager is externally reachable (for example, if # Alertmanager is served via a reverse proxy). Used for generating relative and # absolute links back to Alertmanager itself. If the URL has a path portion, it # will be used to prefix all HTTP endpoints served by Alertmanager. If omitted, # relevant URL components will be derived automatically. # CLI flag: -alertmanager.web.external-url [external_url: \u003curl\u003e | default = ] # How frequently to poll Cortex configs # CLI flag: -alertmanager.configs.poll-interval [poll_interval: \u003cduration\u003e | default = 15s] # Maximum size (bytes) of an accepted HTTP request body. # CLI flag: -alertmanager.max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 16777216] # Shard tenants across multiple alertmanager instances. # CLI flag: -alertmanager.sharding-enabled [sharding_enabled: \u003cboolean\u003e | default = false] sharding_ring: # The key-value store used to share the hash ring across multiple instances. kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -alertmanager.sharding-ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -alertmanager.sharding-ring.prefix [prefix: \u003cstring\u003e | default = \"alertmanagers/\"] dynamodb: # Region to access dynamodb. # CLI flag: -alertmanager.sharding-ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -alertmanager.sharding-ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -alertmanager.sharding-ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -alertmanager.sharding-ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -alertmanager.sharding-ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: alertmanager.sharding-ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: alertmanager.sharding-ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -alertmanager.sharding-ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -alertmanager.sharding-ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -alertmanager.sharding-ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -alertmanager.sharding-ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -alertmanager.sharding-ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 15s] # The heartbeat timeout after which alertmanagers are considered unhealthy # within the ring. 0 = never (timeout disabled). # CLI flag: -alertmanager.sharding-ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # The replication factor to use when sharding the alertmanager. # CLI flag: -alertmanager.sharding-ring.replication-factor [replication_factor: \u003cint\u003e | default = 3] # True to enable zone-awareness and replicate alerts across different # availability zones. # CLI flag: -alertmanager.sharding-ring.zone-awareness-enabled [zone_awareness_enabled: \u003cboolean\u003e | default = false] # The sleep seconds when alertmanager is shutting down. Need to be close to or # larger than KV Store information propagation delay # CLI flag: -alertmanager.sharding-ring.final-sleep [final_sleep: \u003cduration\u003e | default = 0s] # Timeout for waiting on alertmanager to become desired state in the ring. # CLI flag: -alertmanager.sharding-ring.wait-instance-state-timeout [wait_instance_state_timeout: \u003cduration\u003e | default = 10m] # Name of network interface to read address from. # CLI flag: -alertmanager.sharding-ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # The availability zone where this instance is running. Required if # zone-awareness is enabled. # CLI flag: -alertmanager.sharding-ring.instance-availability-zone [instance_availability_zone: \u003cstring\u003e | default = \"\"] # Filename of fallback config to use if none specified for instance. # CLI flag: -alertmanager.configs.fallback [fallback_config_file: \u003cstring\u003e | default = \"\"] # Root of URL to generate if config is http://internal.monitor # CLI flag: -alertmanager.configs.auto-webhook-root [auto_webhook_root: \u003cstring\u003e | default = \"\"] cluster: # Listen address and port for the cluster. Not specifying this flag disables # high-availability mode. # CLI flag: -alertmanager.cluster.listen-address [listen_address: \u003cstring\u003e | default = \"0.0.0.0:9094\"] # Explicit address or hostname to advertise in cluster. # CLI flag: -alertmanager.cluster.advertise-address [advertise_address: \u003cstring\u003e | default = \"\"] # Comma-separated list of initial peers. # CLI flag: -alertmanager.cluster.peers [peers: \u003cstring\u003e | default = \"\"] # Time to wait between peers to send notifications. # CLI flag: -alertmanager.cluster.peer-timeout [peer_timeout: \u003cduration\u003e | default = 15s] # The interval between sending gossip messages. By lowering this value (more # frequent) gossip messages are propagated across cluster more quickly at the # expense of increased bandwidth usage. # CLI flag: -alertmanager.cluster.gossip-interval [gossip_interval: \u003cduration\u003e | default = 200ms] # The interval between gossip state syncs. Setting this interval lower (more # frequent) will increase convergence speeds across larger clusters at the # expense of increased bandwidth usage. # CLI flag: -alertmanager.cluster.push-pull-interval [push_pull_interval: \u003cduration\u003e | default = 1m] # Enable the experimental alertmanager config api. # CLI flag: -experimental.alertmanager.enable-api [enable_api: \u003cboolean\u003e | default = false] # Maximum number of concurrent GET API requests before returning an error. # CLI flag: -alertmanager.api-concurrency [api_concurrency: \u003cint\u003e | default = 0] # Alertmanager alerts Garbage collection interval. # CLI flag: -alertmanager.alerts-gc-interval [gc_interval: \u003cduration\u003e | default = 30m] alertmanager_client: # Timeout for downstream alertmanagers. # CLI flag: -alertmanager.alertmanager-client.remote-timeout [remote_timeout: \u003cduration\u003e | default = 2s] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -alertmanager.alertmanager-client.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -alertmanager.alertmanager-client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -alertmanager.alertmanager-client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -alertmanager.alertmanager-client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -alertmanager.alertmanager-client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -alertmanager.alertmanager-client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy' and '' (disable compression) # CLI flag: -alertmanager.alertmanager-client.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # gRPC client max receive message size (bytes). # CLI flag: -alertmanager.alertmanager-client.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 16777216] # gRPC client max send message size (bytes). # CLI flag: -alertmanager.alertmanager-client.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 4194304] # The interval between persisting the current alertmanager state (notification # log and silences) to object storage. This is only used when sharding is # enabled. This state is read when all replicas for a shard can not be # contacted. In this scenario, having persisted the state more frequently will # result in potentially fewer lost silences, and fewer duplicate notifications. # CLI flag: -alertmanager.persist-interval [persist_interval: \u003cduration\u003e | default = 15m] # Comma separated list of tenants whose alerts this alertmanager can process. If # specified, only these tenants will be handled by alertmanager, otherwise this # alertmanager can process alerts from all tenants. # CLI flag: -alertmanager.enabled-tenants [enabled_tenants: \u003cstring\u003e | default = \"\"] # Comma separated list of tenants whose alerts this alertmanager cannot process. # If specified, a alertmanager that would normally pick the specified tenant(s) # for processing will ignore them instead. # CLI flag: -alertmanager.disabled-tenants [disabled_tenants: \u003cstring\u003e | default = \"\"] alertmanager_storage_config The alertmanager_storage_config configures the Cortex alertmanager storage backend.\n# Backend storage to use. Supported backends are: s3, gcs, azure, swift, # filesystem, configdb, local. # CLI flag: -alertmanager-storage.backend [backend: \u003cstring\u003e | default = \"s3\"] s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an # S3-compatible service in hostname:port format. # CLI flag: -alertmanager-storage.s3.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # S3 region. If unset, the client will issue a S3 GetBucketLocation API call # to autodetect it. # CLI flag: -alertmanager-storage.s3.region [region: \u003cstring\u003e | default = \"\"] # S3 bucket name # CLI flag: -alertmanager-storage.s3.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # S3 secret access key # CLI flag: -alertmanager-storage.s3.secret-access-key [secret_access_key: \u003cstring\u003e | default = \"\"] # S3 access key ID # CLI flag: -alertmanager-storage.s3.access-key-id [access_key_id: \u003cstring\u003e | default = \"\"] # If enabled, use http:// for the S3 endpoint instead of https://. This could # be useful in local dev/test environments while using an S3-compatible # backend storage, like Minio. # CLI flag: -alertmanager-storage.s3.insecure [insecure: \u003cboolean\u003e | default = false] # The signature version to use for authenticating against S3. Supported values # are: v4, v2. # CLI flag: -alertmanager-storage.s3.signature-version [signature_version: \u003cstring\u003e | default = \"v4\"] # The s3 bucket lookup style. Supported values are: auto, virtual-hosted, # path. # CLI flag: -alertmanager-storage.s3.bucket-lookup-type [bucket_lookup_type: \u003cstring\u003e | default = \"auto\"] # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum # algorithm to verify the provided digest. If false, use CRC32C algorithm # instead. # CLI flag: -alertmanager-storage.s3.send-content-md5 [send_content_md5: \u003cboolean\u003e | default = true] # The list api version. Supported values are: v1, v2, and ''. # CLI flag: -alertmanager-storage.s3.list-objects-version [list_objects_version: \u003cstring\u003e | default = \"\"] # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: alertmanager-storage [sse: \u003cs3_sse_config\u003e] http: # The time an idle connection will remain idle before closing. # CLI flag: -alertmanager-storage.s3.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -alertmanager-storage.s3.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -alertmanager-storage.s3.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -alertmanager-storage.s3.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -alertmanager-storage.s3.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -alertmanager-storage.s3.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -alertmanager-storage.s3.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -alertmanager-storage.s3.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] gcs: # GCS bucket name # CLI flag: -alertmanager-storage.gcs.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # JSON representing either a Google Developers Console client_credentials.json # file or a Google Developers service account key file. If empty, fallback to # Google default logic. # CLI flag: -alertmanager-storage.gcs.service-account [service_account: \u003cstring\u003e | default = \"\"] azure: # Azure storage account name # CLI flag: -alertmanager-storage.azure.account-name [account_name: \u003cstring\u003e | default = \"\"] # Azure storage account key # CLI flag: -alertmanager-storage.azure.account-key [account_key: \u003cstring\u003e | default = \"\"] # The values of `account-name` and `endpoint-suffix` values will not be # ignored if `connection-string` is set. Use this method over `account-key` if # you need to authenticate via a SAS token or if you use the Azurite emulator. # CLI flag: -alertmanager-storage.azure.connection-string [connection_string: \u003cstring\u003e | default = \"\"] # Azure storage container name # CLI flag: -alertmanager-storage.azure.container-name [container_name: \u003cstring\u003e | default = \"\"] # Azure storage endpoint suffix without schema. The account name will be # prefixed to this value to create the FQDN # CLI flag: -alertmanager-storage.azure.endpoint-suffix [endpoint_suffix: \u003cstring\u003e | default = \"\"] # Number of retries for recoverable errors # CLI flag: -alertmanager-storage.azure.max-retries [max_retries: \u003cint\u003e | default = 20] # Deprecated: Azure storage MSI resource. It will be set automatically by # Azure SDK. # CLI flag: -alertmanager-storage.azure.msi-resource [msi_resource: \u003cstring\u003e | default = \"\"] # Azure storage MSI resource managed identity client Id. If not supplied # default Azure credential will be used. Set it to empty if you need to # authenticate via Azure Workload Identity. # CLI flag: -alertmanager-storage.azure.user-assigned-id [user_assigned_id: \u003cstring\u003e | default = \"\"] http: # The time an idle connection will remain idle before closing. # CLI flag: -alertmanager-storage.azure.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -alertmanager-storage.azure.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -alertmanager-storage.azure.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -alertmanager-storage.azure.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -alertmanager-storage.azure.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -alertmanager-storage.azure.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -alertmanager-storage.azure.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -alertmanager-storage.azure.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] swift: # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -alertmanager-storage.swift.auth-version [auth_version: \u003cint\u003e | default = 0] # OpenStack Swift authentication URL # CLI flag: -alertmanager-storage.swift.auth-url [auth_url: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential ID. # CLI flag: -alertmanager-storage.swift.application-credential-id [application_credential_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential name. # CLI flag: -alertmanager-storage.swift.application-credential-name [application_credential_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential secret. # CLI flag: -alertmanager-storage.swift.application-credential-secret [application_credential_secret: \u003cstring\u003e | default = \"\"] # OpenStack Swift username. # CLI flag: -alertmanager-storage.swift.username [username: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -alertmanager-storage.swift.user-domain-name [user_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -alertmanager-storage.swift.user-domain-id [user_domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user ID. # CLI flag: -alertmanager-storage.swift.user-id [user_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift API key. # CLI flag: -alertmanager-storage.swift.password [password: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -alertmanager-storage.swift.domain-id [domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -alertmanager-storage.swift.domain-name [domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift project ID (v2,v3 auth only). # CLI flag: -alertmanager-storage.swift.project-id [project_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift project name (v2,v3 auth only). # CLI flag: -alertmanager-storage.swift.project-name [project_name: \u003cstring\u003e | default = \"\"] # ID of the OpenStack Swift project's domain (v3 auth only), only needed if it # differs the from user domain. # CLI flag: -alertmanager-storage.swift.project-domain-id [project_domain_id: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift project's domain (v3 auth only), only needed if # it differs from the user domain. # CLI flag: -alertmanager-storage.swift.project-domain-name [project_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift Region to use (v2,v3 auth only). # CLI flag: -alertmanager-storage.swift.region-name [region_name: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift container to put chunks in. # CLI flag: -alertmanager-storage.swift.container-name [container_name: \u003cstring\u003e | default = \"\"] # Max retries on requests error. # CLI flag: -alertmanager-storage.swift.max-retries [max_retries: \u003cint\u003e | default = 3] # Time after which a connection attempt is aborted. # CLI flag: -alertmanager-storage.swift.connect-timeout [connect_timeout: \u003cduration\u003e | default = 10s] # Time after which an idle request is aborted. The timeout watchdog is reset # each time some data is received, so the timeout triggers after X time no # data is received on a request. # CLI flag: -alertmanager-storage.swift.request-timeout [request_timeout: \u003cduration\u003e | default = 5s] filesystem: # Local filesystem storage directory. # CLI flag: -alertmanager-storage.filesystem.dir [dir: \u003cstring\u003e | default = \"\"] # The configstore_config configures the config database storing rules and # alerts, and is used by the Cortex alertmanager. # The CLI flags prefix for this block config is: alertmanager-storage [configdb: \u003cconfigstore_config\u003e] local: # Path at which alertmanager configurations are stored. # CLI flag: -alertmanager-storage.local.path [path: \u003cstring\u003e | default = \"\"] blocks_storage_config The blocks_storage_config configures the blocks storage.\n# Backend storage to use. Supported backends are: s3, gcs, azure, swift, # filesystem. # CLI flag: -blocks-storage.backend [backend: \u003cstring\u003e | default = \"s3\"] s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an # S3-compatible service in hostname:port format. # CLI flag: -blocks-storage.s3.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # S3 region. If unset, the client will issue a S3 GetBucketLocation API call # to autodetect it. # CLI flag: -blocks-storage.s3.region [region: \u003cstring\u003e | default = \"\"] # S3 bucket name # CLI flag: -blocks-storage.s3.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # S3 secret access key # CLI flag: -blocks-storage.s3.secret-access-key [secret_access_key: \u003cstring\u003e | default = \"\"] # S3 access key ID # CLI flag: -blocks-storage.s3.access-key-id [access_key_id: \u003cstring\u003e | default = \"\"] # If enabled, use http:// for the S3 endpoint instead of https://. This could # be useful in local dev/test environments while using an S3-compatible # backend storage, like Minio. # CLI flag: -blocks-storage.s3.insecure [insecure: \u003cboolean\u003e | default = false] # The signature version to use for authenticating against S3. Supported values # are: v4, v2. # CLI flag: -blocks-storage.s3.signature-version [signature_version: \u003cstring\u003e | default = \"v4\"] # The s3 bucket lookup style. Supported values are: auto, virtual-hosted, # path. # CLI flag: -blocks-storage.s3.bucket-lookup-type [bucket_lookup_type: \u003cstring\u003e | default = \"auto\"] # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum # algorithm to verify the provided digest. If false, use CRC32C algorithm # instead. # CLI flag: -blocks-storage.s3.send-content-md5 [send_content_md5: \u003cboolean\u003e | default = true] # The list api version. Supported values are: v1, v2, and ''. # CLI flag: -blocks-storage.s3.list-objects-version [list_objects_version: \u003cstring\u003e | default = \"\"] # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: blocks-storage [sse: \u003cs3_sse_config\u003e] http: # The time an idle connection will remain idle before closing. # CLI flag: -blocks-storage.s3.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -blocks-storage.s3.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -blocks-storage.s3.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -blocks-storage.s3.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -blocks-storage.s3.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -blocks-storage.s3.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -blocks-storage.s3.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -blocks-storage.s3.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] gcs: # GCS bucket name # CLI flag: -blocks-storage.gcs.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # JSON representing either a Google Developers Console client_credentials.json # file or a Google Developers service account key file. If empty, fallback to # Google default logic. # CLI flag: -blocks-storage.gcs.service-account [service_account: \u003cstring\u003e | default = \"\"] azure: # Azure storage account name # CLI flag: -blocks-storage.azure.account-name [account_name: \u003cstring\u003e | default = \"\"] # Azure storage account key # CLI flag: -blocks-storage.azure.account-key [account_key: \u003cstring\u003e | default = \"\"] # The values of `account-name` and `endpoint-suffix` values will not be # ignored if `connection-string` is set. Use this method over `account-key` if # you need to authenticate via a SAS token or if you use the Azurite emulator. # CLI flag: -blocks-storage.azure.connection-string [connection_string: \u003cstring\u003e | default = \"\"] # Azure storage container name # CLI flag: -blocks-storage.azure.container-name [container_name: \u003cstring\u003e | default = \"\"] # Azure storage endpoint suffix without schema. The account name will be # prefixed to this value to create the FQDN # CLI flag: -blocks-storage.azure.endpoint-suffix [endpoint_suffix: \u003cstring\u003e | default = \"\"] # Number of retries for recoverable errors # CLI flag: -blocks-storage.azure.max-retries [max_retries: \u003cint\u003e | default = 20] # Deprecated: Azure storage MSI resource. It will be set automatically by # Azure SDK. # CLI flag: -blocks-storage.azure.msi-resource [msi_resource: \u003cstring\u003e | default = \"\"] # Azure storage MSI resource managed identity client Id. If not supplied # default Azure credential will be used. Set it to empty if you need to # authenticate via Azure Workload Identity. # CLI flag: -blocks-storage.azure.user-assigned-id [user_assigned_id: \u003cstring\u003e | default = \"\"] http: # The time an idle connection will remain idle before closing. # CLI flag: -blocks-storage.azure.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -blocks-storage.azure.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -blocks-storage.azure.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -blocks-storage.azure.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -blocks-storage.azure.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -blocks-storage.azure.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -blocks-storage.azure.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -blocks-storage.azure.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] swift: # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -blocks-storage.swift.auth-version [auth_version: \u003cint\u003e | default = 0] # OpenStack Swift authentication URL # CLI flag: -blocks-storage.swift.auth-url [auth_url: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential ID. # CLI flag: -blocks-storage.swift.application-credential-id [application_credential_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential name. # CLI flag: -blocks-storage.swift.application-credential-name [application_credential_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential secret. # CLI flag: -blocks-storage.swift.application-credential-secret [application_credential_secret: \u003cstring\u003e | default = \"\"] # OpenStack Swift username. # CLI flag: -blocks-storage.swift.username [username: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -blocks-storage.swift.user-domain-name [user_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -blocks-storage.swift.user-domain-id [user_domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user ID. # CLI flag: -blocks-storage.swift.user-id [user_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift API key. # CLI flag: -blocks-storage.swift.password [password: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -blocks-storage.swift.domain-id [domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -blocks-storage.swift.domain-name [domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift project ID (v2,v3 auth only). # CLI flag: -blocks-storage.swift.project-id [project_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift project name (v2,v3 auth only). # CLI flag: -blocks-storage.swift.project-name [project_name: \u003cstring\u003e | default = \"\"] # ID of the OpenStack Swift project's domain (v3 auth only), only needed if it # differs the from user domain. # CLI flag: -blocks-storage.swift.project-domain-id [project_domain_id: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift project's domain (v3 auth only), only needed if # it differs from the user domain. # CLI flag: -blocks-storage.swift.project-domain-name [project_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift Region to use (v2,v3 auth only). # CLI flag: -blocks-storage.swift.region-name [region_name: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift container to put chunks in. # CLI flag: -blocks-storage.swift.container-name [container_name: \u003cstring\u003e | default = \"\"] # Max retries on requests error. # CLI flag: -blocks-storage.swift.max-retries [max_retries: \u003cint\u003e | default = 3] # Time after which a connection attempt is aborted. # CLI flag: -blocks-storage.swift.connect-timeout [connect_timeout: \u003cduration\u003e | default = 10s] # Time after which an idle request is aborted. The timeout watchdog is reset # each time some data is received, so the timeout triggers after X time no # data is received on a request. # CLI flag: -blocks-storage.swift.request-timeout [request_timeout: \u003cduration\u003e | default = 5s] filesystem: # Local filesystem storage directory. # CLI flag: -blocks-storage.filesystem.dir [dir: \u003cstring\u003e | default = \"\"] # This configures how the querier and store-gateway discover and synchronize # blocks stored in the bucket. bucket_store: # Directory to store synchronized TSDB index headers. # CLI flag: -blocks-storage.bucket-store.sync-dir [sync_dir: \u003cstring\u003e | default = \"tsdb-sync\"] # How frequently to scan the bucket, or to refresh the bucket index (if # enabled), in order to look for changes (new blocks shipped by ingesters and # blocks deleted by retention or compaction). # CLI flag: -blocks-storage.bucket-store.sync-interval [sync_interval: \u003cduration\u003e | default = 15m] # Max number of concurrent queries to execute against the long-term storage. # The limit is shared across all tenants. # CLI flag: -blocks-storage.bucket-store.max-concurrent [max_concurrent: \u003cint\u003e | default = 100] # Max number of inflight queries to execute against the long-term storage. The # limit is shared across all tenants. 0 to disable. # CLI flag: -blocks-storage.bucket-store.max-inflight-requests [max_inflight_requests: \u003cint\u003e | default = 0] # Maximum number of concurrent tenants syncing blocks. # CLI flag: -blocks-storage.bucket-store.tenant-sync-concurrency [tenant_sync_concurrency: \u003cint\u003e | default = 10] # Maximum number of concurrent blocks syncing per tenant. # CLI flag: -blocks-storage.bucket-store.block-sync-concurrency [block_sync_concurrency: \u003cint\u003e | default = 20] # Number of Go routines to use when syncing block meta files from object # storage per tenant. # CLI flag: -blocks-storage.bucket-store.meta-sync-concurrency [meta_sync_concurrency: \u003cint\u003e | default = 20] # Minimum age of a block before it's being read. Set it to safe value (e.g # 30m) if your object storage is eventually consistent. GCS and S3 are # (roughly) strongly consistent. # CLI flag: -blocks-storage.bucket-store.consistency-delay [consistency_delay: \u003cduration\u003e | default = 0s] index_cache: # The index cache backend type. Multiple cache backend can be provided as a # comma-separated ordered list to enable the implementation of a cache # hierarchy. Supported values: inmemory, memcached, redis. # CLI flag: -blocks-storage.bucket-store.index-cache.backend [backend: \u003cstring\u003e | default = \"inmemory\"] inmemory: # Maximum size in bytes of in-memory index cache used to speed up blocks # index lookups (shared between all tenants). # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes [max_size_bytes: \u003cint\u003e | default = 1073741824] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items [enabled_items: \u003clist of string\u003e | default = []] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. If # set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should run. # If more keys are specified, internally keys are split into multiple # batches and fetched concurrently, honoring the max concurrency. If set # to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit breaker # becomes half-open. If set to 0, by default open duration is 60 # seconds. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items [enabled_items: \u003clist of string\u003e | default = []] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.index-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. # If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching is # when data is stored in memory instead of fetching data each time. See # https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit breaker # becomes half-open. If set to 0, by default open duration is 60 # seconds. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items [enabled_items: \u003clist of string\u003e | default = []] multilevel: # The maximum number of concurrent asynchronous operations can occur when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of items to backfill per asynchronous operation. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-backfill-items [max_backfill_items: \u003cint\u003e | default = 10000] chunks_cache: # The chunks cache backend type. Single or Multiple cache backend can be # provided. Supported values in single cache: memcached, redis, inmemory, # and '' (disable). Supported values in multi level cache: a comma-separated # list of (inmemory, memcached, redis) # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend [backend: \u003cstring\u003e | default = \"\"] inmemory: # Maximum size in bytes of in-memory chunk cache used to speed up chunk # lookups (shared between all tenants). # CLI flag: -blocks-storage.bucket-store.chunks-cache.inmemory.max-size-bytes [max_size_bytes: \u003cint\u003e | default = 1073741824] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. If # set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should run. # If more keys are specified, internally keys are split into multiple # batches and fetched concurrently, honoring the max concurrency. If set # to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit breaker # becomes half-open. If set to 0, by default open duration is 60 # seconds. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. # If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching is # when data is stored in memory instead of fetching data each time. See # https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit breaker # becomes half-open. If set to 0, by default open duration is 60 # seconds. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] multilevel: # The maximum number of concurrent asynchronous operations can occur when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of items to backfill per asynchronous operation. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-backfill-items [max_backfill_items: \u003cint\u003e | default = 10000] # Size of each subrange that bucket object is split into for better caching. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-size [subrange_size: \u003cint\u003e | default = 16000] # Maximum number of sub-GetRange requests that a single GetRange request can # be split into when fetching chunks. Zero or negative value = unlimited # number of sub-requests. # CLI flag: -blocks-storage.bucket-store.chunks-cache.max-get-range-requests [max_get_range_requests: \u003cint\u003e | default = 3] # TTL for caching object attributes for chunks. # CLI flag: -blocks-storage.bucket-store.chunks-cache.attributes-ttl [attributes_ttl: \u003cduration\u003e | default = 168h] # TTL for caching individual chunks subranges. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-ttl [subrange_ttl: \u003cduration\u003e | default = 24h] metadata_cache: # Backend for metadata cache, if not empty. Supported values: memcached, # redis, and '' (disable). # CLI flag: -blocks-storage.bucket-store.metadata-cache.backend [backend: \u003cstring\u003e | default = \"\"] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. If # set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should run. # If more keys are specified, internally keys are split into multiple # batches and fetched concurrently, honoring the max concurrency. If set # to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit breaker # becomes half-open. If set to 0, by default open duration is 60 # seconds. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. # If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching is # when data is stored in memory instead of fetching data each time. See # https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit breaker # becomes half-open. If set to 0, by default open duration is 60 # seconds. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # How long to cache list of tenants in the bucket. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl [tenants_list_ttl: \u003cduration\u003e | default = 15m] # How long to cache list of blocks for each tenant. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenant-blocks-list-ttl [tenant_blocks_list_ttl: \u003cduration\u003e | default = 5m] # How long to cache list of chunks for a block. # CLI flag: -blocks-storage.bucket-store.metadata-cache.chunks-list-ttl [chunks_list_ttl: \u003cduration\u003e | default = 24h] # How long to cache information that block metafile exists. Also used for # user deletion mark file. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-exists-ttl [metafile_exists_ttl: \u003cduration\u003e | default = 2h] # How long to cache information that block metafile doesn't exist. Also used # for user deletion mark file. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-doesnt-exist-ttl [metafile_doesnt_exist_ttl: \u003cduration\u003e | default = 5m] # How long to cache content of the metafile. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-content-ttl [metafile_content_ttl: \u003cduration\u003e | default = 24h] # Maximum size of metafile content to cache in bytes. Caching will be # skipped if the content exceeds this size. This is useful to avoid network # round trip for large content if the configured caching backend has an hard # limit on cached items size (in this case, you should set this limit to the # same limit in the caching backend). # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-max-size-bytes [metafile_max_size_bytes: \u003cint\u003e | default = 1048576] # How long to cache attributes of the block metafile. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-attributes-ttl [metafile_attributes_ttl: \u003cduration\u003e | default = 168h] # How long to cache attributes of the block index. # CLI flag: -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl [block_index_attributes_ttl: \u003cduration\u003e | default = 168h] # How long to cache content of the bucket index. # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl [bucket_index_content_ttl: \u003cduration\u003e | default = 5m] # Maximum size of bucket index content to cache in bytes. Caching will be # skipped if the content exceeds this size. This is useful to avoid network # round trip for large content if the configured caching backend has an hard # limit on cached items size (in this case, you should set this limit to the # same limit in the caching backend). # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-max-size-bytes [bucket_index_max_size_bytes: \u003cint\u003e | default = 1048576] # Duration after which the blocks marked for deletion will be filtered out # while fetching blocks. The idea of ignore-deletion-marks-delay is to ignore # blocks that are marked for deletion with some delay. This ensures store can # still serve blocks that are meant to be deleted but do not have a # replacement yet. Default is 6h, half of the default value for # -compactor.deletion-delay. # CLI flag: -blocks-storage.bucket-store.ignore-deletion-marks-delay [ignore_deletion_mark_delay: \u003cduration\u003e | default = 6h] # The blocks created since `now() - ignore_blocks_within` will not be synced. # This should be used together with `-querier.query-store-after` to filter out # the blocks that are too new to be queried. A reasonable value for this flag # would be `-querier.query-store-after - # blocks-storage.bucket-store.bucket-index.max-stale-period` to give some # buffer. 0 to disable. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: \u003cduration\u003e | default = 0s] bucket_index: # True to enable querier and store-gateway to discover blocks in the storage # via bucket index instead of bucket scanning. # CLI flag: -blocks-storage.bucket-store.bucket-index.enabled [enabled: \u003cboolean\u003e | default = false] # How frequently a bucket index, which previously failed to load, should be # tried to load again. This option is used only by querier. # CLI flag: -blocks-storage.bucket-store.bucket-index.update-on-error-interval [update_on_error_interval: \u003cduration\u003e | default = 1m] # How long a unused bucket index should be cached. Once this timeout # expires, the unused bucket index is removed from the in-memory cache. This # option is used only by querier. # CLI flag: -blocks-storage.bucket-store.bucket-index.idle-timeout [idle_timeout: \u003cduration\u003e | default = 1h] # The maximum allowed age of a bucket index (last updated) before queries # start failing because the bucket index is too old. The bucket index is # periodically updated by the compactor, while this check is enforced in the # querier (at query time). # CLI flag: -blocks-storage.bucket-store.bucket-index.max-stale-period [max_stale_period: \u003cduration\u003e | default = 1h] # One of concurrent, recursive, bucket_index. When set to concurrent, stores # will concurrently issue one call per directory to discover active blocks in # the bucket. The recursive strategy iterates through all objects in the # bucket, recursively traversing into each directory. This avoids N+1 calls at # the expense of having slower bucket iterations. bucket_index strategy can be # used in Compactor only and utilizes the existing bucket index to fetch block # IDs to sync. This avoids iterating the bucket but can be impacted by delays # of cleaner creating bucket index. # CLI flag: -blocks-storage.bucket-store.block-discovery-strategy [block_discovery_strategy: \u003cstring\u003e | default = \"concurrent\"] # Max size - in bytes - of a chunks pool, used to reduce memory allocations. # The pool is shared across all tenants. 0 to disable the limit. # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes [max_chunk_pool_bytes: \u003cint\u003e | default = 2147483648] # If enabled, store-gateway will lazily memory-map an index-header only once # required by a query. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-enabled [index_header_lazy_loading_enabled: \u003cboolean\u003e | default = false] # If index-header lazy loading is enabled and this setting is \u003e 0, the # store-gateway will release memory-mapped index-headers after 'idle timeout' # inactivity. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout [index_header_lazy_loading_idle_timeout: \u003cduration\u003e | default = 20m] # If true, Store Gateway will estimate postings size and try to lazily expand # postings if it downloads less data than expanding all postings. # CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled [lazy_expanded_postings_enabled: \u003cboolean\u003e | default = false] # Mark posting group as lazy if it fetches more keys than R * max series the # query should fetch. With R set to 100, a posting group which fetches 100K # keys will be marked as lazy if the current query only fetches 1000 series. # This config is only valid if lazy expanded posting is enabled. 0 disables # the limit. # CLI flag: -blocks-storage.bucket-store.lazy-expanded-posting-group-max-key-series-ratio [lazy_expanded_posting_group_max_key_series_ratio: \u003cfloat\u003e | default = 100] # Controls how many series to fetch per batch in Store Gateway. Default value # is 10000. # CLI flag: -blocks-storage.bucket-store.series-batch-size [series_batch_size: \u003cint\u003e | default = 10000] token_bucket_bytes_limiter: # Token bucket bytes limiter mode. Supported values are: disabled, dryrun, # enabled # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.mode [mode: \u003cstring\u003e | default = \"disabled\"] # Instance token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.instance-token-bucket-size [instance_token_bucket_size: \u003cint\u003e | default = 859832320] # User token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.user-token-bucket-size [user_token_bucket_size: \u003cint\u003e | default = 644874240] # Request token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.request-token-bucket-size [request_token_bucket_size: \u003cint\u003e | default = 4194304] tsdb: # Local directory to store TSDBs in the ingesters. # CLI flag: -blocks-storage.tsdb.dir [dir: \u003cstring\u003e | default = \"tsdb\"] # TSDB blocks range period. # CLI flag: -blocks-storage.tsdb.block-ranges-period [block_ranges_period: \u003clist of duration\u003e | default = 2h0m0s] # TSDB blocks retention in the ingester before a block is removed. This should # be larger than the block_ranges_period and large enough to give # store-gateways and queriers enough time to discover newly uploaded blocks. # CLI flag: -blocks-storage.tsdb.retention-period [retention_period: \u003cduration\u003e | default = 6h] # How frequently the TSDB blocks are scanned and new ones are shipped to the # storage. 0 means shipping is disabled. # CLI flag: -blocks-storage.tsdb.ship-interval [ship_interval: \u003cduration\u003e | default = 1m] # Maximum number of tenants concurrently shipping blocks to the storage. # CLI flag: -blocks-storage.tsdb.ship-concurrency [ship_concurrency: \u003cint\u003e | default = 10] # How frequently does Cortex try to compact TSDB head. Block is only created # if data covers smallest block range. Must be greater than 0 and max 30 # minutes. Note that up to 50% jitter is added to the value for the first # compaction to avoid ingesters compacting concurrently. # CLI flag: -blocks-storage.tsdb.head-compaction-interval [head_compaction_interval: \u003cduration\u003e | default = 1m] # Maximum number of tenants concurrently compacting TSDB head into a new block # CLI flag: -blocks-storage.tsdb.head-compaction-concurrency [head_compaction_concurrency: \u003cint\u003e | default = 5] # If TSDB head is idle for this duration, it is compacted. Note that up to 25% # jitter is added to the value to avoid ingesters compacting concurrently. 0 # means disabled. # CLI flag: -blocks-storage.tsdb.head-compaction-idle-timeout [head_compaction_idle_timeout: \u003cduration\u003e | default = 1h] # The write buffer size used by the head chunks mapper. Lower values reduce # memory utilisation on clusters with a large number of tenants at the cost of # increased disk I/O operations. # CLI flag: -blocks-storage.tsdb.head-chunks-write-buffer-size-bytes [head_chunks_write_buffer_size_bytes: \u003cint\u003e | default = 4194304] # The number of shards of series to use in TSDB (must be a power of 2). # Reducing this will decrease memory footprint, but can negatively impact # performance. # CLI flag: -blocks-storage.tsdb.stripe-size [stripe_size: \u003cint\u003e | default = 16384] # Deprecated (use blocks-storage.tsdb.wal-compression-type instead): True to # enable TSDB WAL compression. # CLI flag: -blocks-storage.tsdb.wal-compression-enabled [wal_compression_enabled: \u003cboolean\u003e | default = false] # TSDB WAL type. Supported values are: 'snappy', 'zstd' and '' (disable # compression) # CLI flag: -blocks-storage.tsdb.wal-compression-type [wal_compression_type: \u003cstring\u003e | default = \"\"] # TSDB WAL segments files max size (bytes). # CLI flag: -blocks-storage.tsdb.wal-segment-size-bytes [wal_segment_size_bytes: \u003cint\u003e | default = 134217728] # True to flush blocks to storage on shutdown. If false, incomplete blocks # will be reused after restart. # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown [flush_blocks_on_shutdown: \u003cboolean\u003e | default = false] # If TSDB has not received any data for this duration, and all blocks from # TSDB have been shipped, TSDB is closed and deleted from local disk. If set # to positive value, this value should be equal or higher than # -querier.query-ingesters-within flag to make sure that TSDB is not closed # prematurely, which could cause partial query results. 0 or negative value # disables closing of idle TSDB. # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout [close_idle_tsdb_timeout: \u003cduration\u003e | default = 0s] # The size of the in-memory queue used before flushing chunks to the disk. # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: \u003cint\u003e | default = 0] # limit the number of concurrently opening TSDB's on startup # CLI flag: -blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup [max_tsdb_opening_concurrency_on_startup: \u003cint\u003e | default = 10] # Deprecated, use maxExemplars in limits instead. If the MaxExemplars value in # limits is set to zero, cortex will fallback on this value. This setting # enables support for exemplars in TSDB and sets the maximum number that will # be stored. 0 or less means disabled. # CLI flag: -blocks-storage.tsdb.max-exemplars [max_exemplars: \u003cint\u003e | default = 0] # True to enable snapshotting of in-memory TSDB data on disk when shutting # down. # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown [memory_snapshot_on_shutdown: \u003cboolean\u003e | default = false] # [EXPERIMENTAL] Configures the maximum number of samples per chunk that can # be out-of-order. # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max [out_of_order_cap_max: \u003cint\u003e | default = 32] # [EXPERIMENTAL] True to enable native histogram. # CLI flag: -blocks-storage.tsdb.enable-native-histograms [enable_native_histograms: \u003cboolean\u003e | default = false] # [EXPERIMENTAL] If enabled, ingesters will cache expanded postings when # querying blocks. Caching can be configured separately for the head and # compacted blocks. expanded_postings_cache: # If enabled, ingesters will cache expanded postings for the head block. # Only queries with with an equal matcher for metric __name__ are cached. head: # Whether the postings cache is enabled or not # CLI flag: -blocks-storage.expanded_postings_cache.head.enabled [enabled: \u003cboolean\u003e | default = false] # Max bytes for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.head.max-bytes [max_bytes: \u003cint\u003e | default = 10485760] # TTL for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.head.ttl [ttl: \u003cduration\u003e | default = 10m] # If enabled, ingesters will cache expanded postings for the compacted # blocks. The cache is shared between all blocks. blocks: # Whether the postings cache is enabled or not # CLI flag: -blocks-storage.expanded_postings_cache.block.enabled [enabled: \u003cboolean\u003e | default = false] # Max bytes for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.block.max-bytes [max_bytes: \u003cint\u003e | default = 10485760] # TTL for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.block.ttl [ttl: \u003cduration\u003e | default = 10m] compactor_config The compactor_config configures the compactor for the blocks storage.\n# List of compaction time ranges. # CLI flag: -compactor.block-ranges [block_ranges: \u003clist of duration\u003e | default = 2h0m0s,12h0m0s,24h0m0s] # Number of Go routines to use when syncing block index and chunks files from # the long term storage. # CLI flag: -compactor.block-sync-concurrency [block_sync_concurrency: \u003cint\u003e | default = 20] # Number of Go routines to use when syncing block meta files from the long term # storage. # CLI flag: -compactor.meta-sync-concurrency [meta_sync_concurrency: \u003cint\u003e | default = 20] # Minimum age of fresh (non-compacted) blocks before they are being processed. # Malformed blocks older than the maximum of consistency-delay and 48h0m0s will # be removed. # CLI flag: -compactor.consistency-delay [consistency_delay: \u003cduration\u003e | default = 0s] # Data directory in which to cache blocks and process compactions # CLI flag: -compactor.data-dir [data_dir: \u003cstring\u003e | default = \"./data\"] # The frequency at which the compaction runs # CLI flag: -compactor.compaction-interval [compaction_interval: \u003cduration\u003e | default = 1h] # How many times to retry a failed compaction within a single compaction run. # CLI flag: -compactor.compaction-retries [compaction_retries: \u003cint\u003e | default = 3] # Max number of concurrent compactions running. # CLI flag: -compactor.compaction-concurrency [compaction_concurrency: \u003cint\u003e | default = 1] # How frequently compactor should run blocks cleanup and maintenance, as well as # update the bucket index. # CLI flag: -compactor.cleanup-interval [cleanup_interval: \u003cduration\u003e | default = 15m] # Max number of tenants for which blocks cleanup and maintenance should run # concurrently. # CLI flag: -compactor.cleanup-concurrency [cleanup_concurrency: \u003cint\u003e | default = 20] # Time before a block marked for deletion is deleted from bucket. If not 0, # blocks will be marked for deletion and compactor component will permanently # delete blocks marked for deletion from the bucket. If 0, blocks will be # deleted straight away. Note that deleting blocks immediately can cause query # failures. # CLI flag: -compactor.deletion-delay [deletion_delay: \u003cduration\u003e | default = 12h] # For tenants marked for deletion, this is time between deleting of last block, # and doing final cleanup (marker files, debug files) of the tenant. # CLI flag: -compactor.tenant-cleanup-delay [tenant_cleanup_delay: \u003cduration\u003e | default = 6h] # When enabled, mark blocks containing index with out-of-order chunks for no # compact instead of halting the compaction. # CLI flag: -compactor.skip-blocks-with-out-of-order-chunks-enabled [skip_blocks_with_out_of_order_chunks_enabled: \u003cboolean\u003e | default = false] # Number of goroutines to use when fetching/uploading block files from object # storage. # CLI flag: -compactor.block-files-concurrency [block_files_concurrency: \u003cint\u003e | default = 10] # Number of goroutines to use when fetching blocks from object storage when # compacting. # CLI flag: -compactor.blocks-fetch-concurrency [blocks_fetch_concurrency: \u003cint\u003e | default = 3] # When enabled, at compactor startup the bucket will be scanned and all found # deletion marks inside the block location will be copied to the markers global # location too. This option can (and should) be safely disabled as soon as the # compactor has successfully run at least once. # CLI flag: -compactor.block-deletion-marks-migration-enabled [block_deletion_marks_migration_enabled: \u003cboolean\u003e | default = false] # Comma separated list of tenants that can be compacted. If specified, only # these tenants will be compacted by compactor, otherwise all tenants can be # compacted. Subject to sharding. # CLI flag: -compactor.enabled-tenants [enabled_tenants: \u003cstring\u003e | default = \"\"] # Comma separated list of tenants that cannot be compacted by this compactor. If # specified, and compactor would normally pick given tenant for compaction (via # -compactor.enabled-tenants or sharding), it will be ignored instead. # CLI flag: -compactor.disabled-tenants [disabled_tenants: \u003cstring\u003e | default = \"\"] # Shard tenants across multiple compactor instances. Sharding is required if you # run multiple compactor instances, in order to coordinate compactions and avoid # race conditions leading to the same tenant blocks simultaneously compacted by # different instances. # CLI flag: -compactor.sharding-enabled [sharding_enabled: \u003cboolean\u003e | default = false] # The sharding strategy to use. Supported values are: default, shuffle-sharding. # CLI flag: -compactor.sharding-strategy [sharding_strategy: \u003cstring\u003e | default = \"default\"] sharding_ring: kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -compactor.ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -compactor.ring.prefix [prefix: \u003cstring\u003e | default = \"collectors/\"] dynamodb: # Region to access dynamodb. # CLI flag: -compactor.ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -compactor.ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -compactor.ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -compactor.ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -compactor.ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: compactor.ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: compactor.ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -compactor.ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -compactor.ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -compactor.ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -compactor.ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -compactor.ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 5s] # The heartbeat timeout after which compactors are considered unhealthy within # the ring. 0 = never (timeout disabled). # CLI flag: -compactor.ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # Minimum time to wait for ring stability at startup. 0 to disable. # CLI flag: -compactor.ring.wait-stability-min-duration [wait_stability_min_duration: \u003cduration\u003e | default = 1m] # Maximum time to wait for ring stability at startup. If the compactor ring # keeps changing after this period of time, the compactor will start anyway. # CLI flag: -compactor.ring.wait-stability-max-duration [wait_stability_max_duration: \u003cduration\u003e | default = 5m] # Name of network interface to read address from. # CLI flag: -compactor.ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # File path where tokens are stored. If empty, tokens are not stored at # shutdown and restored at startup. # CLI flag: -compactor.ring.tokens-file-path [tokens_file_path: \u003cstring\u003e | default = \"\"] # Unregister the compactor during shutdown if true. # CLI flag: -compactor.ring.unregister-on-shutdown [unregister_on_shutdown: \u003cboolean\u003e | default = true] # Timeout for waiting on compactor to become ACTIVE in the ring. # CLI flag: -compactor.ring.wait-active-instance-timeout [wait_active_instance_timeout: \u003cduration\u003e | default = 10m] # The compaction strategy to use. Supported values are: default, partitioning. # CLI flag: -compactor.compaction-mode [compaction_mode: \u003cstring\u003e | default = \"default\"] # How long block visit marker file should be considered as expired and able to # be picked up by compactor again. # CLI flag: -compactor.block-visit-marker-timeout [block_visit_marker_timeout: \u003cduration\u003e | default = 5m] # How frequently block visit marker file should be updated duration compaction. # CLI flag: -compactor.block-visit-marker-file-update-interval [block_visit_marker_file_update_interval: \u003cduration\u003e | default = 1m] # How long cleaner visit marker file should be considered as expired and able to # be picked up by cleaner again. The value should be smaller than # -compactor.cleanup-interval # CLI flag: -compactor.cleaner-visit-marker-timeout [cleaner_visit_marker_timeout: \u003cduration\u003e | default = 10m] # How frequently cleaner visit marker file should be updated when cleaning user. # CLI flag: -compactor.cleaner-visit-marker-file-update-interval [cleaner_visit_marker_file_update_interval: \u003cduration\u003e | default = 5m] # When enabled, index verification will ignore out of order label names. # CLI flag: -compactor.accept-malformed-index [accept_malformed_index: \u003cboolean\u003e | default = false] # When enabled, caching bucket will be used for compactor, except cleaner # service, which serves as the source of truth for block status # CLI flag: -compactor.caching-bucket-enabled [caching_bucket_enabled: \u003cboolean\u003e | default = false] configs_config The configs_config configures the Cortex Configs DB and API.\ndatabase: # URI where the database can be found (for dev you can use memory://) # CLI flag: -configs.database.uri [uri: \u003cstring\u003e | default = \"postgres://postgres@configs-db.weave.local/configs?sslmode=disable\"] # Path where the database migration files can be found # CLI flag: -configs.database.migrations-dir [migrations_dir: \u003cstring\u003e | default = \"\"] # File containing password (username goes in URI) # CLI flag: -configs.database.password-file [password_file: \u003cstring\u003e | default = \"\"] api: notifications: # Disable Email notifications for Alertmanager. # CLI flag: -configs.notifications.disable-email [disable_email: \u003cboolean\u003e | default = false] # Disable WebHook notifications for Alertmanager. # CLI flag: -configs.notifications.disable-webhook [disable_webhook: \u003cboolean\u003e | default = false] configstore_config The configstore_config configures the config database storing rules and alerts, and is used by the Cortex alertmanager. The supported CLI flags \u003cprefix\u003e used to reference this config block are:\nalertmanager-storage ruler-storage # URL of configs API server. # CLI flag: -\u003cprefix\u003e.configs.url [configs_api_url: \u003curl\u003e | default = ] # Timeout for requests to Weave Cloud configs service. # CLI flag: -\u003cprefix\u003e.configs.client-timeout [client_timeout: \u003cduration\u003e | default = 5s] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -\u003cprefix\u003e.configs.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -\u003cprefix\u003e.configs.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -\u003cprefix\u003e.configs.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -\u003cprefix\u003e.configs.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -\u003cprefix\u003e.configs.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] consul_config The consul_config configures the consul client. The supported CLI flags \u003cprefix\u003e used to reference this config block are:\nno prefix alertmanager.sharding-ring compactor.ring distributor.ha-tracker distributor.ring ruler.ring store-gateway.sharding-ring # Hostname and port of Consul. # CLI flag: -\u003cprefix\u003e.consul.hostname [host: \u003cstring\u003e | default = \"localhost:8500\"] # ACL Token used to interact with Consul. # CLI flag: -\u003cprefix\u003e.consul.acl-token [acl_token: \u003cstring\u003e | default = \"\"] # HTTP timeout when talking to Consul. # CLI flag: -\u003cprefix\u003e.consul.client-timeout [http_client_timeout: \u003cduration\u003e | default = 20s] # Enable consistent reads to Consul. # CLI flag: -\u003cprefix\u003e.consul.consistent-reads [consistent_reads: \u003cboolean\u003e | default = false] # Rate limit when watching key or prefix in Consul, in requests per second. 0 # disables the rate limit. # CLI flag: -\u003cprefix\u003e.consul.watch-rate-limit [watch_rate_limit: \u003cfloat\u003e | default = 1] # Burst size used in rate limit. Values less than 1 are treated as 1. # CLI flag: -\u003cprefix\u003e.consul.watch-burst-size [watch_burst_size: \u003cint\u003e | default = 1] # Enable TLS. # CLI flag: -\u003cprefix\u003e.consul.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -\u003cprefix\u003e.consul.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -\u003cprefix\u003e.consul.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -\u003cprefix\u003e.consul.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -\u003cprefix\u003e.consul.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -\u003cprefix\u003e.consul.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] distributor_config The distributor_config configures the Cortex distributor.\npool: # How frequently to clean up clients for ingesters that have gone away. # CLI flag: -distributor.client-cleanup-period [client_cleanup_period: \u003cduration\u003e | default = 15s] # Run a health check on each ingester client during periodic cleanup. # CLI flag: -distributor.health-check-ingesters [health_check_ingesters: \u003cboolean\u003e | default = true] ha_tracker: # Enable the distributors HA tracker so that it can accept samples from # Prometheus HA replicas gracefully (requires labels). # CLI flag: -distributor.ha-tracker.enable [enable_ha_tracker: \u003cboolean\u003e | default = false] # Update the timestamp in the KV store for a given cluster/replica only after # this amount of time has passed since the current stored timestamp. # CLI flag: -distributor.ha-tracker.update-timeout [ha_tracker_update_timeout: \u003cduration\u003e | default = 15s] # Maximum jitter applied to the update timeout, in order to spread the HA # heartbeats over time. # CLI flag: -distributor.ha-tracker.update-timeout-jitter-max [ha_tracker_update_timeout_jitter_max: \u003cduration\u003e | default = 5s] # If we don't receive any samples from the accepted replica for a cluster in # this amount of time we will failover to the next replica we receive a sample # from. This value must be greater than the update timeout # CLI flag: -distributor.ha-tracker.failover-timeout [ha_tracker_failover_timeout: \u003cduration\u003e | default = 30s] # Backend storage to use for the ring. Please be aware that memberlist is not # supported by the HA tracker since gossip propagation is too slow for HA # purposes. kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -distributor.ha-tracker.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -distributor.ha-tracker.prefix [prefix: \u003cstring\u003e | default = \"ha-tracker/\"] dynamodb: # Region to access dynamodb. # CLI flag: -distributor.ha-tracker.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -distributor.ha-tracker.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -distributor.ha-tracker.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -distributor.ha-tracker.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -distributor.ha-tracker.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: distributor.ha-tracker [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: distributor.ha-tracker [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -distributor.ha-tracker.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -distributor.ha-tracker.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -distributor.ha-tracker.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -distributor.ha-tracker.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # remote_write API max receive message size (bytes). # CLI flag: -distributor.max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # Maximum OTLP request size in bytes that the Distributor can accept. # CLI flag: -distributor.otlp-max-recv-msg-size [otlp_max_recv_msg_size: \u003cint\u003e | default = 104857600] # Timeout for downstream ingesters. # CLI flag: -distributor.remote-timeout [remote_timeout: \u003cduration\u003e | default = 2s] # Time to wait before sending more than the minimum successful query requests. # CLI flag: -distributor.extra-query-delay [extra_queue_delay: \u003cduration\u003e | default = 0s] # The sharding strategy to use. Supported values are: default, shuffle-sharding. # CLI flag: -distributor.sharding-strategy [sharding_strategy: \u003cstring\u003e | default = \"default\"] # Distribute samples based on all labels, as opposed to solely by user and # metric name. # CLI flag: -distributor.shard-by-all-labels [shard_by_all_labels: \u003cboolean\u003e | default = false] # Try writing to an additional ingester in the presence of an ingester not in # the ACTIVE state. It is useful to disable this along with # -ingester.unregister-on-shutdown=false in order to not spread samples to extra # ingesters during rolling restarts with consistent naming. # CLI flag: -distributor.extend-writes [extend_writes: \u003cboolean\u003e | default = true] # EXPERIMENTAL: If enabled, sign the write request between distributors and # ingesters. # CLI flag: -distributor.sign-write-requests [sign_write_requests: \u003cboolean\u003e | default = false] ring: kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -distributor.ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -distributor.ring.prefix [prefix: \u003cstring\u003e | default = \"collectors/\"] dynamodb: # Region to access dynamodb. # CLI flag: -distributor.ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -distributor.ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -distributor.ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -distributor.ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -distributor.ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: distributor.ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: distributor.ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -distributor.ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -distributor.ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -distributor.ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -distributor.ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -distributor.ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 5s] # The heartbeat timeout after which distributors are considered unhealthy # within the ring. 0 = never (timeout disabled). # CLI flag: -distributor.ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # Name of network interface to read address from. # CLI flag: -distributor.ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # EXPERIMENTAL: Number of go routines to handle push calls from distributors to # ingesters. When no workers are available, a new goroutine will be spawned # automatically. If set to 0 (default), workers are disabled, and a new # goroutine will be created for each push request. # CLI flag: -distributor.num-push-workers [num_push_workers: \u003cint\u003e | default = 0] instance_limits: # Max ingestion rate (samples/sec) that this distributor will accept. This # limit is per-distributor, not per-tenant. Additional push requests will be # rejected. Current ingestion rate is computed as exponentially weighted # moving average, updated every second. 0 = unlimited. # CLI flag: -distributor.instance-limits.max-ingestion-rate [max_ingestion_rate: \u003cfloat\u003e | default = 0] # Max inflight push requests that this distributor can handle. This limit is # per-distributor, not per-tenant. Additional requests will be rejected. 0 = # unlimited. # CLI flag: -distributor.instance-limits.max-inflight-push-requests [max_inflight_push_requests: \u003cint\u003e | default = 0] # Max inflight ingester client requests that this distributor can handle. This # limit is per-distributor, not per-tenant. Additional requests will be # rejected. 0 = unlimited. # CLI flag: -distributor.instance-limits.max-inflight-client-requests [max_inflight_client_requests: \u003cint\u003e | default = 0] otlp: # If true, all resource attributes are converted to labels. # CLI flag: -distributor.otlp.convert-all-attributes [convert_all_attributes: \u003cboolean\u003e | default = false] # If true, a target_info metric is not ingested. (refer to: # https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#supporting-target-metadata-in-both-push-based-and-pull-based-systems) # CLI flag: -distributor.otlp.disable-target-info [disable_target_info: \u003cboolean\u003e | default = false] etcd_config The etcd_config configures the etcd client. The supported CLI flags \u003cprefix\u003e used to reference this config block are:\nno prefix alertmanager.sharding-ring compactor.ring distributor.ha-tracker distributor.ring ruler.ring store-gateway.sharding-ring # The etcd endpoints to connect to. # CLI flag: -\u003cprefix\u003e.etcd.endpoints [endpoints: \u003clist of string\u003e | default = []] # The dial timeout for the etcd connection. # CLI flag: -\u003cprefix\u003e.etcd.dial-timeout [dial_timeout: \u003cduration\u003e | default = 10s] # The maximum number of retries to do for failed ops. # CLI flag: -\u003cprefix\u003e.etcd.max-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS. # CLI flag: -\u003cprefix\u003e.etcd.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -\u003cprefix\u003e.etcd.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -\u003cprefix\u003e.etcd.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -\u003cprefix\u003e.etcd.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -\u003cprefix\u003e.etcd.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -\u003cprefix\u003e.etcd.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # Etcd username. # CLI flag: -\u003cprefix\u003e.etcd.username [username: \u003cstring\u003e | default = \"\"] # Etcd password. # CLI flag: -\u003cprefix\u003e.etcd.password [password: \u003cstring\u003e | default = \"\"] # Send Keepalive pings with no streams. # CLI flag: -\u003cprefix\u003e.etcd.ping-without-stream-allowed [ping-without-stream-allowed: \u003cboolean\u003e | default = true] fifo_cache_config The fifo_cache_config configures the local in-memory cache.\n# Maximum memory size of the cache in bytes. A unit suffix (KB, MB, GB) may be # applied. # CLI flag: -frontend.fifocache.max-size-bytes [max_size_bytes: \u003cstring\u003e | default = \"\"] # Maximum number of entries in the cache. # CLI flag: -frontend.fifocache.max-size-items [max_size_items: \u003cint\u003e | default = 0] # The expiry duration for the cache. # CLI flag: -frontend.fifocache.duration [validity: \u003cduration\u003e | default = 0s] # Deprecated (use max-size-items or max-size-bytes instead): The number of # entries to cache. # CLI flag: -frontend.fifocache.size [size: \u003cint\u003e | default = 0] flusher_config The flusher_config configures the WAL flusher target, used to manually run one-time flushes when scaling down ingesters.\n# Stop Cortex after flush has finished. If false, Cortex process will keep # running, doing nothing. # CLI flag: -flusher.exit-after-flush [exit_after_flush: \u003cboolean\u003e | default = true] frontend_worker_config The frontend_worker_config configures the worker - running within the Cortex querier - picking up and executing queries enqueued by the query-frontend or query-scheduler.\n# Address of query frontend service, in host:port format. If # -querier.scheduler-address is set as well, querier will use scheduler instead. # Only one of -querier.frontend-address or -querier.scheduler-address can be # set. If neither is set, queries are only received via HTTP endpoint. # CLI flag: -querier.frontend-address [frontend_address: \u003cstring\u003e | default = \"\"] # Hostname (and port) of scheduler that querier will periodically resolve, # connect to and receive queries from. Only one of -querier.frontend-address or # -querier.scheduler-address can be set. If neither is set, queries are only # received via HTTP endpoint. # CLI flag: -querier.scheduler-address [scheduler_address: \u003cstring\u003e | default = \"\"] # How often to query DNS for query-frontend or query-scheduler address. # CLI flag: -querier.dns-lookup-period [dns_lookup_duration: \u003cduration\u003e | default = 10s] # Number of simultaneous queries to process per query-frontend or # query-scheduler. # CLI flag: -querier.worker-parallelism [parallelism: \u003cint\u003e | default = 10] # Force worker concurrency to match the -querier.max-concurrent option. # Overrides querier.worker-parallelism. # CLI flag: -querier.worker-match-max-concurrent [match_max_concurrent: \u003cboolean\u003e | default = false] # Querier ID, sent to frontend service to identify requests from the same # querier. Defaults to hostname. # CLI flag: -querier.id [id: \u003cstring\u003e | default = \"\"] grpc_client_config: # gRPC client max receive message size (bytes). # CLI flag: -querier.frontend-client.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # gRPC client max send message size (bytes). # CLI flag: -querier.frontend-client.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 16777216] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy', 'snappy-block' ,'zstd' and '' (disable compression) # CLI flag: -querier.frontend-client.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # Rate limit for gRPC client; 0 means disabled. # CLI flag: -querier.frontend-client.grpc-client-rate-limit [rate_limit: \u003cfloat\u003e | default = 0] # Rate limit burst for gRPC client. # CLI flag: -querier.frontend-client.grpc-client-rate-limit-burst [rate_limit_burst: \u003cint\u003e | default = 0] # Enable backoff and retry when we hit ratelimits. # CLI flag: -querier.frontend-client.backoff-on-ratelimits [backoff_on_ratelimits: \u003cboolean\u003e | default = false] backoff_config: # Minimum delay when backing off. # CLI flag: -querier.frontend-client.backoff-min-period [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -querier.frontend-client.backoff-max-period [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -querier.frontend-client.backoff-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -querier.frontend-client.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -querier.frontend-client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -querier.frontend-client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -querier.frontend-client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -querier.frontend-client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -querier.frontend-client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] ingester_config The ingester_config configures the Cortex ingester.\nlifecycler: ring: kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -ring.prefix [prefix: \u003cstring\u003e | default = \"collectors/\"] dynamodb: # Region to access dynamodb. # CLI flag: -dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # The heartbeat timeout after which ingesters are skipped for reads/writes. # 0 = never (timeout disabled). # CLI flag: -ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # The number of ingesters to write to and read from. # CLI flag: -distributor.replication-factor [replication_factor: \u003cint\u003e | default = 3] # True to enable the zone-awareness and replicate ingested samples across # different availability zones. # CLI flag: -distributor.zone-awareness-enabled [zone_awareness_enabled: \u003cboolean\u003e | default = false] # Comma-separated list of zones to exclude from the ring. Instances in # excluded zones will be filtered out from the ring. # CLI flag: -distributor.excluded-zones [excluded_zones: \u003cstring\u003e | default = \"\"] # Set to true to enable ring detailed metrics. These metrics provide # detailed information, such as token count and ownership per tenant. # Disabling them can significantly decrease the number of metrics emitted by # the distributors. # CLI flag: -ring.detailed-metrics-enabled [detailed_metrics_enabled: \u003cboolean\u003e | default = true] # Number of tokens for each ingester. # CLI flag: -ingester.num-tokens [num_tokens: \u003cint\u003e | default = 128] # EXPERIMENTAL: Algorithm used to generate new ring tokens. Supported Values: # random,minimize-spread # CLI flag: -ingester.tokens-generator-strategy [tokens_generator_strategy: \u003cstring\u003e | default = \"random\"] # Period at which to heartbeat to consul. 0 = disabled. # CLI flag: -ingester.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 5s] # Observe tokens after generating to resolve collisions. Useful when using # gossiping ring. # CLI flag: -ingester.observe-period [observe_period: \u003cduration\u003e | default = 0s] # Period to wait for a claim from another member; will join automatically # after this. # CLI flag: -ingester.join-after [join_after: \u003cduration\u003e | default = 0s] # Minimum duration to wait after the internal readiness checks have passed but # before succeeding the readiness endpoint. This is used to slowdown # deployment controllers (eg. Kubernetes) after an instance is ready and # before they proceed with a rolling update, to give the rest of the cluster # instances enough time to receive ring updates. # CLI flag: -ingester.min-ready-duration [min_ready_duration: \u003cduration\u003e | default = 15s] # Name of network interface to read address from. # CLI flag: -ingester.lifecycler.interface [interface_names: \u003clist of string\u003e | default = [eth0 en0]] # Duration to sleep for before exiting, to ensure metrics are scraped. # CLI flag: -ingester.final-sleep [final_sleep: \u003cduration\u003e | default = 30s] # File path where tokens are stored. If empty, tokens are not stored at # shutdown and restored at startup. # CLI flag: -ingester.tokens-file-path [tokens_file_path: \u003cstring\u003e | default = \"\"] # The availability zone where this instance is running. # CLI flag: -ingester.availability-zone [availability_zone: \u003cstring\u003e | default = \"\"] # Unregister from the ring upon clean shutdown. It can be useful to disable # for rolling restarts with consistent naming in conjunction with # -distributor.extend-writes=false. # CLI flag: -ingester.unregister-on-shutdown [unregister_on_shutdown: \u003cboolean\u003e | default = true] # When enabled the readiness probe succeeds only after all instances are # ACTIVE and healthy in the ring, otherwise only the instance itself is # checked. This option should be disabled if in your cluster multiple # instances can be rolled out simultaneously, otherwise rolling updates may be # slowed down. # CLI flag: -ingester.readiness-check-ring-health [readiness_check_ring_health: \u003cboolean\u003e | default = true] # Period at which metadata we have not seen will remain in memory before being # deleted. # CLI flag: -ingester.metadata-retain-period [metadata_retain_period: \u003cduration\u003e | default = 10m] # Period with which to update the per-user ingestion rates. # CLI flag: -ingester.rate-update-period [rate_update_period: \u003cduration\u003e | default = 15s] # Period with which to update the per-user tsdb config. # CLI flag: -ingester.user-tsdb-configs-update-period [user_tsdb_configs_update_period: \u003cduration\u003e | default = 15s] # Enable tracking of active series and export them as metrics. # CLI flag: -ingester.active-series-metrics-enabled [active_series_metrics_enabled: \u003cboolean\u003e | default = true] # How often to update active series metrics. # CLI flag: -ingester.active-series-metrics-update-period [active_series_metrics_update_period: \u003cduration\u003e | default = 1m] # After what time a series is considered to be inactive. # CLI flag: -ingester.active-series-metrics-idle-timeout [active_series_metrics_idle_timeout: \u003cduration\u003e | default = 10m] # Enable uploading compacted blocks. # CLI flag: -ingester.upload-compacted-blocks-enabled [upload_compacted_blocks_enabled: \u003cboolean\u003e | default = true] instance_limits: # Max ingestion rate (samples/sec) that ingester will accept. This limit is # per-ingester, not per-tenant. Additional push requests will be rejected. # Current ingestion rate is computed as exponentially weighted moving average, # updated every second. This limit only works when using blocks engine. 0 = # unlimited. # CLI flag: -ingester.instance-limits.max-ingestion-rate [max_ingestion_rate: \u003cfloat\u003e | default = 0] # Max users that this ingester can hold. Requests from additional users will # be rejected. This limit only works when using blocks engine. 0 = unlimited. # CLI flag: -ingester.instance-limits.max-tenants [max_tenants: \u003cint\u003e | default = 0] # Max series that this ingester can hold (across all tenants). Requests to # create additional series will be rejected. This limit only works when using # blocks engine. 0 = unlimited. # CLI flag: -ingester.instance-limits.max-series [max_series: \u003cint\u003e | default = 0] # Max inflight push requests that this ingester can handle (across all # tenants). Additional requests will be rejected. 0 = unlimited. # CLI flag: -ingester.instance-limits.max-inflight-push-requests [max_inflight_push_requests: \u003cint\u003e | default = 0] # Max inflight query requests that this ingester can handle (across all # tenants). Additional requests will be rejected. 0 = unlimited. # CLI flag: -ingester.instance-limits.max-inflight-query-requests [max_inflight_query_requests: \u003cint\u003e | default = 0] # Comma-separated list of metric names, for which # -ingester.max-series-per-metric and -ingester.max-global-series-per-metric # limits will be ignored. Does not affect max-series-per-user or # max-global-series-per-metric limits. # CLI flag: -ingester.ignore-series-limit-for-metric-names [ignore_series_limit_for_metric_names: \u003cstring\u003e | default = \"\"] # Customize the message contained in limit errors # CLI flag: -ingester.admin-limit-message [admin_limit_message: \u003cstring\u003e | default = \"please contact administrator to raise it\"] # Experimental: Enable string interning for metrics labels. # CLI flag: -ingester.labels-string-interning-enabled [labels_string_interning_enabled: \u003cboolean\u003e | default = false] # Disable trimming of matching series chunks based on query Start and End time. # When disabled, the result may contain samples outside the queried time range # but select performances may be improved. Note that certain query results might # change by changing this option. # CLI flag: -ingester.disable-chunk-trimming [disable_chunk_trimming: \u003cboolean\u003e | default = false] ingester_client_config The ingester_client_config configures how the Cortex distributors connect to the ingesters.\ngrpc_client_config: # gRPC client max receive message size (bytes). # CLI flag: -ingester.client.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # gRPC client max send message size (bytes). # CLI flag: -ingester.client.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 16777216] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy', 'snappy-block' ,'zstd' and '' (disable compression) # CLI flag: -ingester.client.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"snappy-block\"] # Rate limit for gRPC client; 0 means disabled. # CLI flag: -ingester.client.grpc-client-rate-limit [rate_limit: \u003cfloat\u003e | default = 0] # Rate limit burst for gRPC client. # CLI flag: -ingester.client.grpc-client-rate-limit-burst [rate_limit_burst: \u003cint\u003e | default = 0] # Enable backoff and retry when we hit ratelimits. # CLI flag: -ingester.client.backoff-on-ratelimits [backoff_on_ratelimits: \u003cboolean\u003e | default = false] backoff_config: # Minimum delay when backing off. # CLI flag: -ingester.client.backoff-min-period [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -ingester.client.backoff-max-period [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -ingester.client.backoff-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -ingester.client.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -ingester.client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -ingester.client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -ingester.client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -ingester.client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -ingester.client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # EXPERIMENTAL: If enabled, gRPC clients perform health checks for each target # and fail the request if the target is marked as unhealthy. healthcheck_config: # The number of consecutive failed health checks required before considering # a target unhealthy. 0 means disabled. # CLI flag: -ingester.client.healthcheck.unhealthy-threshold [unhealthy_threshold: \u003cint\u003e | default = 0] # The approximate amount of time between health checks of an individual # target. # CLI flag: -ingester.client.healthcheck.interval [interval: \u003cduration\u003e | default = 5s] # The amount of time during which no response from a target means a failed # health check. # CLI flag: -ingester.client.healthcheck.timeout [timeout: \u003cduration\u003e | default = 1s] # Max inflight push requests that this ingester client can handle. This limit is # per-ingester-client. Additional requests will be rejected. 0 = unlimited. # CLI flag: -ingester.client.max-inflight-push-requests [max_inflight_push_requests: \u003cint\u003e | default = 0] limits_config The limits_config configures default and per-tenant limits imposed by Cortex services (ie. distributor, ingester, …).\n# Per-user ingestion rate limit in samples per second. # CLI flag: -distributor.ingestion-rate-limit [ingestion_rate: \u003cfloat\u003e | default = 25000] # Whether the ingestion rate limit should be applied individually to each # distributor instance (local), or evenly shared across the cluster (global). # CLI flag: -distributor.ingestion-rate-limit-strategy [ingestion_rate_strategy: \u003cstring\u003e | default = \"local\"] # Per-user allowed ingestion burst size (in number of samples). # CLI flag: -distributor.ingestion-burst-size [ingestion_burst_size: \u003cint\u003e | default = 50000] # Flag to enable, for all users, handling of samples with external labels # identifying replicas in an HA Prometheus setup. # CLI flag: -distributor.ha-tracker.enable-for-all-users [accept_ha_samples: \u003cboolean\u003e | default = false] # [Experimental] Flag to enable handling of samples with mixed external labels # identifying replicas in an HA Prometheus setup. Supported only if # -distributor.ha-tracker.enable-for-all-users is true. # CLI flag: -experimental.distributor.ha-tracker.mixed-ha-samples [accept_mixed_ha_samples: \u003cboolean\u003e | default = false] # Prometheus label to look for in samples to identify a Prometheus HA cluster. # CLI flag: -distributor.ha-tracker.cluster [ha_cluster_label: \u003cstring\u003e | default = \"cluster\"] # Prometheus label to look for in samples to identify a Prometheus HA replica. # CLI flag: -distributor.ha-tracker.replica [ha_replica_label: \u003cstring\u003e | default = \"__replica__\"] # Maximum number of clusters that HA tracker will keep track of for single user. # 0 to disable the limit. # CLI flag: -distributor.ha-tracker.max-clusters [ha_max_clusters: \u003cint\u003e | default = 0] # This flag can be used to specify label names that to drop during sample # ingestion within the distributor and can be repeated in order to drop multiple # labels. # CLI flag: -distributor.drop-label [drop_labels: \u003clist of string\u003e | default = []] # Maximum length accepted for label names # CLI flag: -validation.max-length-label-name [max_label_name_length: \u003cint\u003e | default = 1024] # Maximum length accepted for label value. This setting also applies to the # metric name # CLI flag: -validation.max-length-label-value [max_label_value_length: \u003cint\u003e | default = 2048] # Maximum number of label names per series. # CLI flag: -validation.max-label-names-per-series [max_label_names_per_series: \u003cint\u003e | default = 30] # Maximum combined size in bytes of all labels and label values accepted for a # series. 0 to disable the limit. # CLI flag: -validation.max-labels-size-bytes [max_labels_size_bytes: \u003cint\u003e | default = 0] # Maximum length accepted for metric metadata. Metadata refers to Metric Name, # HELP and UNIT. # CLI flag: -validation.max-metadata-length [max_metadata_length: \u003cint\u003e | default = 1024] # Reject old samples. # CLI flag: -validation.reject-old-samples [reject_old_samples: \u003cboolean\u003e | default = false] # Maximum accepted sample age before rejecting. # CLI flag: -validation.reject-old-samples.max-age [reject_old_samples_max_age: \u003cduration\u003e | default = 2w] # Duration which table will be created/deleted before/after it's needed; we # won't accept sample from before this time. # CLI flag: -validation.create-grace-period [creation_grace_period: \u003cduration\u003e | default = 10m] # Enforce every metadata has a metric name. # CLI flag: -validation.enforce-metadata-metric-name [enforce_metadata_metric_name: \u003cboolean\u003e | default = true] # Enforce every sample has a metric name. # CLI flag: -validation.enforce-metric-name [enforce_metric_name: \u003cboolean\u003e | default = true] # The default tenant's shard size when the shuffle-sharding strategy is used. # Must be set both on ingesters and distributors. When this setting is specified # in the per-tenant overrides, a value of 0 disables shuffle sharding for the # tenant. # CLI flag: -distributor.ingestion-tenant-shard-size [ingestion_tenant_shard_size: \u003cint\u003e | default = 0] # List of metric relabel configurations. Note that in most situations, it is # more effective to use metrics relabeling directly in the Prometheus server, # e.g. remote_write.write_relabel_configs. [metric_relabel_configs: \u003crelabel_config...\u003e | default = []] # Limit on total number of positive and negative buckets allowed in a single # native histogram. The resolution of a histogram with more buckets will be # reduced until the number of buckets is within the limit. If the limit cannot # be reached, the sample will be discarded. 0 means no limit. Enforced at # Distributor. # CLI flag: -validation.max-native-histogram-buckets [max_native_histogram_buckets: \u003cint\u003e | default = 0] # Comma separated list of resource attributes that should be converted to # labels. # CLI flag: -distributor.promote-resource-attributes [promote_resource_attributes: \u003clist of string\u003e | default = ] # The maximum number of active series per user, per ingester. 0 to disable. # CLI flag: -ingester.max-series-per-user [max_series_per_user: \u003cint\u003e | default = 5000000] # The maximum number of active series per metric name, per ingester. 0 to # disable. # CLI flag: -ingester.max-series-per-metric [max_series_per_metric: \u003cint\u003e | default = 50000] # The maximum number of active series per user, across the cluster before # replication. 0 to disable. Supported only if -distributor.shard-by-all-labels # is true. # CLI flag: -ingester.max-global-series-per-user [max_global_series_per_user: \u003cint\u003e | default = 0] # The maximum number of active series per metric name, across the cluster before # replication. 0 to disable. # CLI flag: -ingester.max-global-series-per-metric [max_global_series_per_metric: \u003cint\u003e | default = 0] # [Experimental] Enable limits per LabelSet. Supported limits per labelSet: # [max_series] [limits_per_label_set: \u003clist of LimitsPerLabelSet\u003e | default = []] # The maximum number of active metrics with metadata per user, per ingester. 0 # to disable. # CLI flag: -ingester.max-metadata-per-user [max_metadata_per_user: \u003cint\u003e | default = 8000] # The maximum number of metadata per metric, per ingester. 0 to disable. # CLI flag: -ingester.max-metadata-per-metric [max_metadata_per_metric: \u003cint\u003e | default = 10] # The maximum number of active metrics with metadata per user, across the # cluster. 0 to disable. Supported only if -distributor.shard-by-all-labels is # true. # CLI flag: -ingester.max-global-metadata-per-user [max_global_metadata_per_user: \u003cint\u003e | default = 0] # The maximum number of metadata per metric, across the cluster. 0 to disable. # CLI flag: -ingester.max-global-metadata-per-metric [max_global_metadata_per_metric: \u003cint\u003e | default = 0] # [Experimental] Configures the allowed time window for ingestion of # out-of-order samples. Disabled (0s) by default. # CLI flag: -ingester.out-of-order-time-window [out_of_order_time_window: \u003cduration\u003e | default = 0s] # Enables support for exemplars in TSDB and sets the maximum number that will be # stored. less than zero means disabled. If the value is set to zero, cortex # will fallback to blocks-storage.tsdb.max-exemplars value. # CLI flag: -ingester.max-exemplars [max_exemplars: \u003cint\u003e | default = 0] # Maximum number of chunks that can be fetched in a single query from ingesters # and long-term storage. This limit is enforced in the querier, ruler and # store-gateway. 0 to disable. # CLI flag: -querier.max-fetched-chunks-per-query [max_fetched_chunks_per_query: \u003cint\u003e | default = 2000000] # The maximum number of unique series for which a query can fetch samples from # each ingesters and blocks storage. This limit is enforced in the querier, # ruler and store-gateway. 0 to disable # CLI flag: -querier.max-fetched-series-per-query [max_fetched_series_per_query: \u003cint\u003e | default = 0] # Deprecated (use max-fetched-data-bytes-per-query instead): The maximum size of # all chunks in bytes that a query can fetch from each ingester and storage. # This limit is enforced in the querier, ruler and store-gateway. 0 to disable. # CLI flag: -querier.max-fetched-chunk-bytes-per-query [max_fetched_chunk_bytes_per_query: \u003cint\u003e | default = 0] # The maximum combined size of all data that a query can fetch from each # ingester and storage. This limit is enforced in the querier and ruler for # `query`, `query_range` and `series` APIs. 0 to disable. # CLI flag: -querier.max-fetched-data-bytes-per-query [max_fetched_data_bytes_per_query: \u003cint\u003e | default = 0] # Limit how long back data (series and metadata) can be queried, up until # \u003clookback\u003e duration ago. This limit is enforced in the query-frontend, querier # and ruler. If the requested time range is outside the allowed range, the # request will not fail but will be manipulated to only query data within the # allowed time range. 0 to disable. # CLI flag: -querier.max-query-lookback [max_query_lookback: \u003cduration\u003e | default = 0s] # Limit the query time range (end - start time of range query parameter and max # - min of data fetched time range). This limit is enforced in the # query-frontend and ruler (on the received query). 0 to disable. # CLI flag: -store.max-query-length [max_query_length: \u003cduration\u003e | default = 0s] # Maximum number of split queries will be scheduled in parallel by the frontend. # CLI flag: -querier.max-query-parallelism [max_query_parallelism: \u003cint\u003e | default = 14] # Most recent allowed cacheable result per-tenant, to prevent caching very # recent results that might still be in flux. # CLI flag: -frontend.max-cache-freshness [max_cache_freshness: \u003cduration\u003e | default = 1m] # Maximum number of queriers that can handle requests for a single tenant. If # set to 0 or value higher than number of available queriers, *all* queriers # will handle requests for the tenant. If the value is \u003c 1, it will be treated # as a percentage and the gets a percentage of the total queriers. Each frontend # (or query-scheduler, if used) will select the same set of queriers for the # same tenant (given that all queriers are connected to all frontends / # query-schedulers). This option only works with queriers connecting to the # query-frontend / query-scheduler, not when using downstream URL. # CLI flag: -frontend.max-queriers-per-tenant [max_queriers_per_tenant: \u003cfloat\u003e | default = 0] # Maximum number of outstanding requests per tenant per request queue (either # query frontend or query scheduler); requests beyond this error with HTTP 429. # CLI flag: -frontend.max-outstanding-requests-per-tenant [max_outstanding_requests_per_tenant: \u003cint\u003e | default = 100] # Configuration for query priority. query_priority: # Whether queries are assigned with priorities. # CLI flag: -frontend.query-priority.enabled [enabled: \u003cboolean\u003e | default = false] # Priority assigned to all queries by default. Must be a unique value. Use # this as a baseline to make certain queries higher/lower priority. # CLI flag: -frontend.query-priority.default-priority [default_priority: \u003cint\u003e | default = 0] # List of priority definitions. [priorities: \u003clist of PriorityDef\u003e | default = []] # Configuration for query rejection. query_rejection: # Whether query rejection is enabled. # CLI flag: -frontend.query-rejection.enabled [enabled: \u003cboolean\u003e | default = false] # List of query_attributes to match and reject queries. A query is rejected if # it matches any query_attribute in this list. Each query_attribute has # several properties (e.g., regex, time_window, user_agent), and all specified # properties must match for a query_attribute to be considered a match. Only # the specified properties are checked, and an AND operator is applied to # them. [query_attributes: \u003clist of QueryAttribute\u003e | default = []] # Deprecated(use ruler.query-offset instead) and will be removed in v1.19.0: # Duration to delay the evaluation of rules to ensure the underlying metrics # have been pushed to Cortex. # CLI flag: -ruler.evaluation-delay-duration [ruler_evaluation_delay_duration: \u003cduration\u003e | default = 0s] # The default tenant's shard size when the shuffle-sharding strategy is used by # ruler. When this setting is specified in the per-tenant overrides, a value of # 0 disables shuffle sharding for the tenant. # CLI flag: -ruler.tenant-shard-size [ruler_tenant_shard_size: \u003cint\u003e | default = 0] # Maximum number of rules per rule group per-tenant. 0 to disable. # CLI flag: -ruler.max-rules-per-rule-group [ruler_max_rules_per_rule_group: \u003cint\u003e | default = 0] # Maximum number of rule groups per-tenant. 0 to disable. # CLI flag: -ruler.max-rule-groups-per-tenant [ruler_max_rule_groups_per_tenant: \u003cint\u003e | default = 0] # Duration to offset all rule evaluation queries per-tenant. # CLI flag: -ruler.query-offset [ruler_query_offset: \u003cduration\u003e | default = 0s] # external labels for alerting rules [ruler_external_labels: \u003cmap of string (labelName) to string (labelValue)\u003e | default = []] # The default tenant's shard size when the shuffle-sharding strategy is used. # Must be set when the store-gateway sharding is enabled with the # shuffle-sharding strategy. When this setting is specified in the per-tenant # overrides, a value of 0 disables shuffle sharding for the tenant. If the value # is \u003c 1 the shard size will be a percentage of the total store-gateways. # CLI flag: -store-gateway.tenant-shard-size [store_gateway_tenant_shard_size: \u003cfloat\u003e | default = 0] # The maximum number of data bytes to download per gRPC request in Store # Gateway, including Series/LabelNames/LabelValues requests. 0 to disable. # CLI flag: -store-gateway.max-downloaded-bytes-per-request [max_downloaded_bytes_per_request: \u003cint\u003e | default = 0] # Delete blocks containing samples older than the specified retention period. 0 # to disable. # CLI flag: -compactor.blocks-retention-period [compactor_blocks_retention_period: \u003cduration\u003e | default = 0s] # The default tenant's shard size when the shuffle-sharding strategy is used by # the compactor. When this setting is specified in the per-tenant overrides, a # value of 0 disables shuffle sharding for the tenant. # CLI flag: -compactor.tenant-shard-size [compactor_tenant_shard_size: \u003cint\u003e | default = 0] # S3 server-side encryption type. Required to enable server-side encryption # overrides for a specific tenant. If not set, the default S3 client settings # are used. [s3_sse_type: \u003cstring\u003e | default = \"\"] # S3 server-side encryption KMS Key ID. Ignored if the SSE type override is not # set. [s3_sse_kms_key_id: \u003cstring\u003e | default = \"\"] # S3 server-side encryption KMS encryption context. If unset and the key ID # override is set, the encryption context will not be provided to S3. Ignored if # the SSE type override is not set. [s3_sse_kms_encryption_context: \u003cstring\u003e | default = \"\"] # Comma-separated list of network CIDRs to block in Alertmanager receiver # integrations. # CLI flag: -alertmanager.receivers-firewall-block-cidr-networks [alertmanager_receivers_firewall_block_cidr_networks: \u003cstring\u003e | default = \"\"] # True to block private and local addresses in Alertmanager receiver # integrations. It blocks private addresses defined by RFC 1918 (IPv4 # addresses) and RFC 4193 (IPv6 addresses), as well as loopback, local unicast # and local multicast addresses. # CLI flag: -alertmanager.receivers-firewall-block-private-addresses [alertmanager_receivers_firewall_block_private_addresses: \u003cboolean\u003e | default = false] # Per-user rate limit for sending notifications from Alertmanager in # notifications/sec. 0 = rate limit disabled. Negative value = no notifications # are allowed. # CLI flag: -alertmanager.notification-rate-limit [alertmanager_notification_rate_limit: \u003cfloat\u003e | default = 0] # Per-integration notification rate limits. Value is a map, where each key is # integration name and value is a rate-limit (float). On command line, this map # is given in JSON format. Rate limit has the same meaning as # -alertmanager.notification-rate-limit, but only applies for specific # integration. Allowed integration names: webhook, email, pagerduty, opsgenie, # wechat, slack, victorops, pushover, sns, telegram, discord, webex, msteams. # CLI flag: -alertmanager.notification-rate-limit-per-integration [alertmanager_notification_rate_limit_per_integration: \u003cmap of string to float64\u003e | default = {}] # Maximum size of configuration file for Alertmanager that tenant can upload via # Alertmanager API. 0 = no limit. # CLI flag: -alertmanager.max-config-size-bytes [alertmanager_max_config_size_bytes: \u003cint\u003e | default = 0] # Maximum number of templates in tenant's Alertmanager configuration uploaded # via Alertmanager API. 0 = no limit. # CLI flag: -alertmanager.max-templates-count [alertmanager_max_templates_count: \u003cint\u003e | default = 0] # Maximum size of single template in tenant's Alertmanager configuration # uploaded via Alertmanager API. 0 = no limit. # CLI flag: -alertmanager.max-template-size-bytes [alertmanager_max_template_size_bytes: \u003cint\u003e | default = 0] # Maximum number of aggregation groups in Alertmanager's dispatcher that a # tenant can have. Each active aggregation group uses single goroutine. When the # limit is reached, dispatcher will not dispatch alerts that belong to # additional aggregation groups, but existing groups will keep working properly. # 0 = no limit. # CLI flag: -alertmanager.max-dispatcher-aggregation-groups [alertmanager_max_dispatcher_aggregation_groups: \u003cint\u003e | default = 0] # Maximum number of alerts that a single user can have. Inserting more alerts # will fail with a log message and metric increment. 0 = no limit. # CLI flag: -alertmanager.max-alerts-count [alertmanager_max_alerts_count: \u003cint\u003e | default = 0] # Maximum total size of alerts that a single user can have, alert size is the # sum of the bytes of its labels, annotations and generatorURL. Inserting more # alerts will fail with a log message and metric increment. 0 = no limit. # CLI flag: -alertmanager.max-alerts-size-bytes [alertmanager_max_alerts_size_bytes: \u003cint\u003e | default = 0] # list of rule groups to disable [disabled_rule_groups: \u003clist of DisabledRuleGroup\u003e | default = []] memberlist_config The memberlist_config configures the Gossip memberlist.\n# Name of the node in memberlist cluster. Defaults to hostname. # CLI flag: -memberlist.nodename [node_name: \u003cstring\u003e | default = \"\"] # Add random suffix to the node name. # CLI flag: -memberlist.randomize-node-name [randomize_node_name: \u003cboolean\u003e | default = true] # The timeout for establishing a connection with a remote node, and for # read/write operations. # CLI flag: -memberlist.stream-timeout [stream_timeout: \u003cduration\u003e | default = 10s] # Multiplication factor used when sending out messages (factor * log(N+1)). # CLI flag: -memberlist.retransmit-factor [retransmit_factor: \u003cint\u003e | default = 4] # How often to use pull/push sync. # CLI flag: -memberlist.pullpush-interval [pull_push_interval: \u003cduration\u003e | default = 30s] # How often to gossip. # CLI flag: -memberlist.gossip-interval [gossip_interval: \u003cduration\u003e | default = 200ms] # How many nodes to gossip to. # CLI flag: -memberlist.gossip-nodes [gossip_nodes: \u003cint\u003e | default = 3] # How long to keep gossiping to dead nodes, to give them chance to refute their # death. # CLI flag: -memberlist.gossip-to-dead-nodes-time [gossip_to_dead_nodes_time: \u003cduration\u003e | default = 30s] # How soon can dead node's name be reclaimed with new address. 0 to disable. # CLI flag: -memberlist.dead-node-reclaim-time [dead_node_reclaim_time: \u003cduration\u003e | default = 0s] # Enable message compression. This can be used to reduce bandwidth usage at the # cost of slightly more CPU utilization. # CLI flag: -memberlist.compression-enabled [compression_enabled: \u003cboolean\u003e | default = true] # Gossip address to advertise to other members in the cluster. Used for NAT # traversal. # CLI flag: -memberlist.advertise-addr [advertise_addr: \u003cstring\u003e | default = \"\"] # Gossip port to advertise to other members in the cluster. Used for NAT # traversal. # CLI flag: -memberlist.advertise-port [advertise_port: \u003cint\u003e | default = 7946] # Other cluster members to join. Can be specified multiple times. It can be an # IP, hostname or an entry specified in the DNS Service Discovery format. # CLI flag: -memberlist.join [join_members: \u003clist of string\u003e | default = []] # Min backoff duration to join other cluster members. # CLI flag: -memberlist.min-join-backoff [min_join_backoff: \u003cduration\u003e | default = 1s] # Max backoff duration to join other cluster members. # CLI flag: -memberlist.max-join-backoff [max_join_backoff: \u003cduration\u003e | default = 1m] # Max number of retries to join other cluster members. # CLI flag: -memberlist.max-join-retries [max_join_retries: \u003cint\u003e | default = 10] # If this node fails to join memberlist cluster, abort. # CLI flag: -memberlist.abort-if-join-fails [abort_if_cluster_join_fails: \u003cboolean\u003e | default = true] # If not 0, how often to rejoin the cluster. Occasional rejoin can help to fix # the cluster split issue, and is harmless otherwise. For example when using # only few components as a seed nodes (via -memberlist.join), then it's # recommended to use rejoin. If -memberlist.join points to dynamic service that # resolves to all gossiping nodes (eg. Kubernetes headless service), then rejoin # is not needed. # CLI flag: -memberlist.rejoin-interval [rejoin_interval: \u003cduration\u003e | default = 0s] # How long to keep LEFT ingesters in the ring. # CLI flag: -memberlist.left-ingesters-timeout [left_ingesters_timeout: \u003cduration\u003e | default = 5m] # Timeout for leaving memberlist cluster. # CLI flag: -memberlist.leave-timeout [leave_timeout: \u003cduration\u003e | default = 5s] # How much space to use for keeping received and sent messages in memory for # troubleshooting (two buffers). 0 to disable. # CLI flag: -memberlist.message-history-buffer-bytes [message_history_buffer_bytes: \u003cint\u003e | default = 0] # IP address to listen on for gossip messages. Multiple addresses may be # specified. Defaults to 0.0.0.0 # CLI flag: -memberlist.bind-addr [bind_addr: \u003clist of string\u003e | default = []] # Port to listen on for gossip messages. # CLI flag: -memberlist.bind-port [bind_port: \u003cint\u003e | default = 7946] # Timeout used when connecting to other nodes to send packet. # CLI flag: -memberlist.packet-dial-timeout [packet_dial_timeout: \u003cduration\u003e | default = 5s] # Timeout for writing 'packet' data. # CLI flag: -memberlist.packet-write-timeout [packet_write_timeout: \u003cduration\u003e | default = 5s] # Enable TLS on the memberlist transport layer. # CLI flag: -memberlist.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -memberlist.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -memberlist.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -memberlist.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -memberlist.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -memberlist.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] memcached_config The memcached_config block configures how data is stored in Memcached (ie. expiration).\n# How long keys stay in the memcache. # CLI flag: -frontend.memcached.expiration [expiration: \u003cduration\u003e | default = 0s] # How many keys to fetch in each batch. # CLI flag: -frontend.memcached.batchsize [batch_size: \u003cint\u003e | default = 1024] # Maximum active requests to memcache. # CLI flag: -frontend.memcached.parallelism [parallelism: \u003cint\u003e | default = 100] memcached_client_config The memcached_client_config configures the client used to connect to Memcached.\n# Hostname for memcached service to use. If empty and if addresses is unset, no # memcached will be used. # CLI flag: -frontend.memcached.hostname [host: \u003cstring\u003e | default = \"\"] # SRV service used to discover memcache servers. # CLI flag: -frontend.memcached.service [service: \u003cstring\u003e | default = \"memcached\"] # EXPERIMENTAL: Comma separated addresses list in DNS Service Discovery format: # https://cortexmetrics.io/docs/configuration/arguments/#dns-service-discovery # CLI flag: -frontend.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # Maximum time to wait before giving up on memcached requests. # CLI flag: -frontend.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # Maximum number of idle connections in pool. # CLI flag: -frontend.memcached.max-idle-conns [max_idle_conns: \u003cint\u003e | default = 16] # The maximum size of an item stored in memcached. Bigger items are not stored. # If set to 0, no maximum size is enforced. # CLI flag: -frontend.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 0] # Period with which to poll DNS for memcache servers. # CLI flag: -frontend.memcached.update-interval [update_interval: \u003cduration\u003e | default = 1m] # Use consistent hashing to distribute to memcache servers. # CLI flag: -frontend.memcached.consistent-hash [consistent_hash: \u003cboolean\u003e | default = true] # Trip circuit-breaker after this number of consecutive dial failures (if zero # then circuit-breaker is disabled). # CLI flag: -frontend.memcached.circuit-breaker-consecutive-failures [circuit_breaker_consecutive_failures: \u003cint\u003e | default = 10] # Duration circuit-breaker remains open after tripping (if zero then 60 seconds # is used). # CLI flag: -frontend.memcached.circuit-breaker-timeout [circuit_breaker_timeout: \u003cduration\u003e | default = 10s] # Reset circuit-breaker counts after this long (if zero then never reset). # CLI flag: -frontend.memcached.circuit-breaker-interval [circuit_breaker_interval: \u003cduration\u003e | default = 10s] querier_config The querier_config configures the Cortex querier.\n# The maximum number of concurrent queries. # CLI flag: -querier.max-concurrent [max_concurrent: \u003cint\u003e | default = 20] # The timeout for a query. # CLI flag: -querier.timeout [timeout: \u003cduration\u003e | default = 2m] # Deprecated (This feature will be always on after v1.18): Use streaming RPCs # for metadata APIs from ingester. # CLI flag: -querier.ingester-metadata-streaming [ingester_metadata_streaming: \u003cboolean\u003e | default = true] # Use LabelNames ingester RPCs with match params. # CLI flag: -querier.ingester-label-names-with-matchers [ingester_label_names_with_matchers: \u003cboolean\u003e | default = false] # Maximum number of samples a single query can load into memory. # CLI flag: -querier.max-samples [max_samples: \u003cint\u003e | default = 50000000] # Maximum lookback beyond which queries are not sent to ingester. 0 means all # queries are sent to ingester. # CLI flag: -querier.query-ingesters-within [query_ingesters_within: \u003cduration\u003e | default = 0s] # Enable returning samples stats per steps in query response. # CLI flag: -querier.per-step-stats-enabled [per_step_stats_enabled: \u003cboolean\u003e | default = false] # Use compression for metrics query API or instant and range query APIs. # Supports 'gzip' and '' (disable compression) # CLI flag: -querier.response-compression [response_compression: \u003cstring\u003e | default = \"gzip\"] # The time after which a metric should be queried from storage and not just # ingesters. 0 means all queries are sent to store. When running the blocks # storage, if this option is enabled, the time range of the query sent to the # store will be manipulated to ensure the query end is not more recent than 'now # - query-store-after'. # CLI flag: -querier.query-store-after [query_store_after: \u003cduration\u003e | default = 0s] # Maximum duration into the future you can query. 0 to disable. # CLI flag: -querier.max-query-into-future [max_query_into_future: \u003cduration\u003e | default = 10m] # The default evaluation interval or step size for subqueries. # CLI flag: -querier.default-evaluation-interval [default_evaluation_interval: \u003cduration\u003e | default = 1m] # Max number of steps allowed for every subquery expression in query. Number of # steps is calculated using subquery range / step. A value \u003e 0 enables it. # CLI flag: -querier.max-subquery-steps [max_subquery_steps: \u003cint\u003e | default = 0] # Active query tracker monitors active queries, and writes them to the file in # given directory. If Cortex discovers any queries in this log during startup, # it will log them to the log file. Setting to empty value disables active query # tracker, which also disables -querier.max-concurrent option. # CLI flag: -querier.active-query-tracker-dir [active_query_tracker_dir: \u003cstring\u003e | default = \"./active-query-tracker\"] # Time since the last sample after which a time series is considered stale and # ignored by expression evaluations. # CLI flag: -querier.lookback-delta [lookback_delta: \u003cduration\u003e | default = 5m] # Comma separated list of store-gateway addresses in DNS Service Discovery # format. This option should be set when using the blocks storage and the # store-gateway sharding is disabled (when enabled, the store-gateway instances # form a ring and addresses are picked from the ring). # CLI flag: -querier.store-gateway-addresses [store_gateway_addresses: \u003cstring\u003e | default = \"\"] store_gateway_client: # Enable TLS for gRPC client connecting to store-gateway. # CLI flag: -querier.store-gateway-client.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -querier.store-gateway-client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -querier.store-gateway-client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -querier.store-gateway-client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -querier.store-gateway-client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -querier.store-gateway-client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy' and '' (disable compression) # CLI flag: -querier.store-gateway-client.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # EXPERIMENTAL: If enabled, gRPC clients perform health checks for each target # and fail the request if the target is marked as unhealthy. healthcheck_config: # The number of consecutive failed health checks required before considering # a target unhealthy. 0 means disabled. # CLI flag: -querier.store-gateway-client.healthcheck.unhealthy-threshold [unhealthy_threshold: \u003cint\u003e | default = 0] # The approximate amount of time between health checks of an individual # target. # CLI flag: -querier.store-gateway-client.healthcheck.interval [interval: \u003cduration\u003e | default = 5s] # The amount of time during which no response from a target means a failed # health check. # CLI flag: -querier.store-gateway-client.healthcheck.timeout [timeout: \u003cduration\u003e | default = 1s] # If enabled, store gateway query stats will be logged using `info` log level. # CLI flag: -querier.store-gateway-query-stats-enabled [store_gateway_query_stats: \u003cboolean\u003e | default = true] # The maximum number of times we attempt fetching missing blocks from different # store-gateways. If no more store-gateways are left (ie. due to lower # replication factor) than we'll end the retries earlier # CLI flag: -querier.store-gateway-consistency-check-max-attempts [store_gateway_consistency_check_max_attempts: \u003cint\u003e | default = 3] # When distributor's sharding strategy is shuffle-sharding and this setting is \u003e # 0, queriers fetch in-memory series from the minimum set of required ingesters, # selecting only ingesters which may have received series since 'now - lookback # period'. The lookback period should be greater or equal than the configured # 'query store after' and 'query ingesters within'. If this setting is 0, # queriers always query all ingesters (ingesters shuffle sharding on read path # is disabled). # CLI flag: -querier.shuffle-sharding-ingesters-lookback-period [shuffle_sharding_ingesters_lookback_period: \u003cduration\u003e | default = 0s] # Experimental. Use Thanos promql engine # https://github.com/thanos-io/promql-engine rather than the Prometheus promql # engine. # CLI flag: -querier.thanos-engine [thanos_engine: \u003cboolean\u003e | default = false] # If enabled, ignore max query length check at Querier select method. Users can # choose to ignore it since the validation can be done before Querier evaluation # like at Query Frontend or Ruler. # CLI flag: -querier.ignore-max-query-length [ignore_max_query_length: \u003cboolean\u003e | default = false] # [Experimental] If true, experimental promQL functions are enabled. # CLI flag: -querier.enable-promql-experimental-functions [enable_promql_experimental_functions: \u003cboolean\u003e | default = false] query_frontend_config The query_frontend_config configures the Cortex query-frontend.\n# Log queries that are slower than the specified duration. Set to 0 to disable. # Set to \u003c 0 to enable on all queries. # CLI flag: -frontend.log-queries-longer-than [log_queries_longer_than: \u003cduration\u003e | default = 0s] # Max body size for downstream prometheus. # CLI flag: -frontend.max-body-size [max_body_size: \u003cint\u003e | default = 10485760] # True to enable query statistics tracking. When enabled, a message with some # statistics is logged for every query. # CLI flag: -frontend.query-stats-enabled [query_stats_enabled: \u003cboolean\u003e | default = false] # If a querier disconnects without sending notification about graceful shutdown, # the query-frontend will keep the querier in the tenant's shard until the # forget delay has passed. This feature is useful to reduce the blast radius # when shuffle-sharding is enabled. # CLI flag: -query-frontend.querier-forget-delay [querier_forget_delay: \u003cduration\u003e | default = 0s] # DNS hostname used for finding query-schedulers. # CLI flag: -frontend.scheduler-address [scheduler_address: \u003cstring\u003e | default = \"\"] # How often to resolve the scheduler-address, in order to look for new # query-scheduler instances. # CLI flag: -frontend.scheduler-dns-lookup-period [scheduler_dns_lookup_period: \u003cduration\u003e | default = 10s] # Number of concurrent workers forwarding queries to single query-scheduler. # CLI flag: -frontend.scheduler-worker-concurrency [scheduler_worker_concurrency: \u003cint\u003e | default = 5] grpc_client_config: # gRPC client max receive message size (bytes). # CLI flag: -frontend.grpc-client-config.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # gRPC client max send message size (bytes). # CLI flag: -frontend.grpc-client-config.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 16777216] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy', 'snappy-block' ,'zstd' and '' (disable compression) # CLI flag: -frontend.grpc-client-config.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # Rate limit for gRPC client; 0 means disabled. # CLI flag: -frontend.grpc-client-config.grpc-client-rate-limit [rate_limit: \u003cfloat\u003e | default = 0] # Rate limit burst for gRPC client. # CLI flag: -frontend.grpc-client-config.grpc-client-rate-limit-burst [rate_limit_burst: \u003cint\u003e | default = 0] # Enable backoff and retry when we hit ratelimits. # CLI flag: -frontend.grpc-client-config.backoff-on-ratelimits [backoff_on_ratelimits: \u003cboolean\u003e | default = false] backoff_config: # Minimum delay when backing off. # CLI flag: -frontend.grpc-client-config.backoff-min-period [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -frontend.grpc-client-config.backoff-max-period [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -frontend.grpc-client-config.backoff-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -frontend.grpc-client-config.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -frontend.grpc-client-config.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -frontend.grpc-client-config.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -frontend.grpc-client-config.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -frontend.grpc-client-config.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -frontend.grpc-client-config.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # When multiple query-schedulers are available, re-enqueue queries that were # rejected due to too many outstanding requests. # CLI flag: -frontend.retry-on-too-many-outstanding-requests [retry_on_too_many_outstanding_requests: \u003cboolean\u003e | default = false] # Name of network interface to read address from. This address is sent to # query-scheduler and querier, which uses it to send the query response back to # query-frontend. # CLI flag: -frontend.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # URL of downstream Prometheus. # CLI flag: -frontend.downstream-url [downstream_url: \u003cstring\u003e | default = \"\"] query_range_config The query_range_config configures the query splitting and caching in the Cortex query-frontend.\n# Split queries by an interval and execute in parallel, 0 disables it. You # should use a multiple of 24 hours (same as the storage bucketing scheme), to # avoid queriers downloading and processing the same chunks. This also # determines how cache keys are chosen when result caching is enabled # CLI flag: -querier.split-queries-by-interval [split_queries_by_interval: \u003cduration\u003e | default = 0s] # Mutate incoming queries to align their start and end with their step. # CLI flag: -querier.align-querier-with-step [align_queries_with_step: \u003cboolean\u003e | default = false] results_cache: cache: # Enable in-memory cache. # CLI flag: -frontend.cache.enable-fifocache [enable_fifocache: \u003cboolean\u003e | default = false] # The default validity of entries for caches unless overridden. # CLI flag: -frontend.default-validity [default_validity: \u003cduration\u003e | default = 0s] background: # At what concurrency to write back to cache. # CLI flag: -frontend.background.write-back-concurrency [writeback_goroutines: \u003cint\u003e | default = 10] # How many key batches to buffer for background write-back. # CLI flag: -frontend.background.write-back-buffer [writeback_buffer: \u003cint\u003e | default = 10000] # The memcached_config block configures how data is stored in Memcached (ie. # expiration). [memcached: \u003cmemcached_config\u003e] # The memcached_client_config configures the client used to connect to # Memcached. [memcached_client: \u003cmemcached_client_config\u003e] # The redis_config configures the Redis backend cache. [redis: \u003credis_config\u003e] # The fifo_cache_config configures the local in-memory cache. [fifocache: \u003cfifo_cache_config\u003e] # Use compression in results cache. Supported values are: 'snappy' and '' # (disable compression). # CLI flag: -frontend.compression [compression: \u003cstring\u003e | default = \"\"] # Cache Statistics queryable samples on results cache. # CLI flag: -frontend.cache-queryable-samples-stats [cache_queryable_samples_stats: \u003cboolean\u003e | default = false] # Cache query results. # CLI flag: -querier.cache-results [cache_results: \u003cboolean\u003e | default = false] # Maximum number of retries for a single request; beyond this, the downstream # error is returned. # CLI flag: -querier.max-retries-per-request [max_retries: \u003cint\u003e | default = 5] # List of headers forwarded by the query Frontend to downstream querier. # CLI flag: -frontend.forward-headers-list [forward_headers_list: \u003clist of string\u003e | default = []] redis_config The redis_config configures the Redis backend cache.\n# Redis Server endpoint to use for caching. A comma-separated list of endpoints # for Redis Cluster or Redis Sentinel. If empty, no redis will be used. # CLI flag: -frontend.redis.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # Redis Sentinel master name. An empty string for Redis Server or Redis Cluster. # CLI flag: -frontend.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # Maximum time to wait before giving up on redis requests. # CLI flag: -frontend.redis.timeout [timeout: \u003cduration\u003e | default = 500ms] # How long keys stay in the redis. # CLI flag: -frontend.redis.expiration [expiration: \u003cduration\u003e | default = 0s] # Database index. # CLI flag: -frontend.redis.db [db: \u003cint\u003e | default = 0] # Maximum number of connections in the pool. # CLI flag: -frontend.redis.pool-size [pool_size: \u003cint\u003e | default = 0] # Password to use when connecting to redis. # CLI flag: -frontend.redis.password [password: \u003cstring\u003e | default = \"\"] # Enable connecting to redis with TLS. # CLI flag: -frontend.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Skip validating server certificate. # CLI flag: -frontend.redis.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # Close connections after remaining idle for this duration. If the value is # zero, then idle connections are not closed. # CLI flag: -frontend.redis.idle-timeout [idle_timeout: \u003cduration\u003e | default = 0s] # Close connections older than this duration. If the value is zero, then the # pool does not close connections based on age. # CLI flag: -frontend.redis.max-connection-age [max_connection_age: \u003cduration\u003e | default = 0s] ruler_config The ruler_config configures the Cortex ruler.\n# [Experimental] GRPC listen address of the Query Frontend, in host:port format. # If set, Ruler queries to Query Frontends via gRPC. If not set, ruler queries # to Ingesters directly. # CLI flag: -ruler.frontend-address [frontend_address: \u003cstring\u003e | default = \"\"] frontend_client: # gRPC client max receive message size (bytes). # CLI flag: -ruler.frontendClient.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # gRPC client max send message size (bytes). # CLI flag: -ruler.frontendClient.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 16777216] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy', 'snappy-block' ,'zstd' and '' (disable compression) # CLI flag: -ruler.frontendClient.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # Rate limit for gRPC client; 0 means disabled. # CLI flag: -ruler.frontendClient.grpc-client-rate-limit [rate_limit: \u003cfloat\u003e | default = 0] # Rate limit burst for gRPC client. # CLI flag: -ruler.frontendClient.grpc-client-rate-limit-burst [rate_limit_burst: \u003cint\u003e | default = 0] # Enable backoff and retry when we hit ratelimits. # CLI flag: -ruler.frontendClient.backoff-on-ratelimits [backoff_on_ratelimits: \u003cboolean\u003e | default = false] backoff_config: # Minimum delay when backing off. # CLI flag: -ruler.frontendClient.backoff-min-period [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -ruler.frontendClient.backoff-max-period [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -ruler.frontendClient.backoff-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -ruler.frontendClient.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -ruler.frontendClient.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -ruler.frontendClient.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -ruler.frontendClient.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -ruler.frontendClient.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -ruler.frontendClient.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # URL of alerts return path. # CLI flag: -ruler.external.url [external_url: \u003curl\u003e | default = ] # Labels to add to all alerts. [external_labels: \u003cmap of string (labelName) to string (labelValue)\u003e | default = []] ruler_client: # gRPC client max receive message size (bytes). # CLI flag: -ruler.client.grpc-max-recv-msg-size [max_recv_msg_size: \u003cint\u003e | default = 104857600] # gRPC client max send message size (bytes). # CLI flag: -ruler.client.grpc-max-send-msg-size [max_send_msg_size: \u003cint\u003e | default = 16777216] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy', 'snappy-block' ,'zstd' and '' (disable compression) # CLI flag: -ruler.client.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # Rate limit for gRPC client; 0 means disabled. # CLI flag: -ruler.client.grpc-client-rate-limit [rate_limit: \u003cfloat\u003e | default = 0] # Rate limit burst for gRPC client. # CLI flag: -ruler.client.grpc-client-rate-limit-burst [rate_limit_burst: \u003cint\u003e | default = 0] # Enable backoff and retry when we hit ratelimits. # CLI flag: -ruler.client.backoff-on-ratelimits [backoff_on_ratelimits: \u003cboolean\u003e | default = false] backoff_config: # Minimum delay when backing off. # CLI flag: -ruler.client.backoff-min-period [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -ruler.client.backoff-max-period [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -ruler.client.backoff-retries [max_retries: \u003cint\u003e | default = 10] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -ruler.client.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -ruler.client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -ruler.client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -ruler.client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -ruler.client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -ruler.client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # Timeout for downstream rulers. # CLI flag: -ruler.client.remote-timeout [remote_timeout: \u003cduration\u003e | default = 2m] # How frequently to evaluate rules # CLI flag: -ruler.evaluation-interval [evaluation_interval: \u003cduration\u003e | default = 1m] # How frequently to poll for rule changes # CLI flag: -ruler.poll-interval [poll_interval: \u003cduration\u003e | default = 1m] # file path to store temporary rule files for the prometheus rule managers # CLI flag: -ruler.rule-path [rule_path: \u003cstring\u003e | default = \"/rules\"] # Comma-separated list of URL(s) of the Alertmanager(s) to send notifications # to. Each Alertmanager URL is treated as a separate group in the configuration. # Multiple Alertmanagers in HA per group can be supported by using DNS # resolution via -ruler.alertmanager-discovery. # CLI flag: -ruler.alertmanager-url [alertmanager_url: \u003cstring\u003e | default = \"\"] # Use DNS SRV records to discover Alertmanager hosts. # CLI flag: -ruler.alertmanager-discovery [enable_alertmanager_discovery: \u003cboolean\u003e | default = false] # How long to wait between refreshing DNS resolutions of Alertmanager hosts. # CLI flag: -ruler.alertmanager-refresh-interval [alertmanager_refresh_interval: \u003cduration\u003e | default = 1m] # Capacity of the queue for notifications to be sent to the Alertmanager. # CLI flag: -ruler.notification-queue-capacity [notification_queue_capacity: \u003cint\u003e | default = 10000] # HTTP timeout duration when sending notifications to the Alertmanager. # CLI flag: -ruler.notification-timeout [notification_timeout: \u003cduration\u003e | default = 10s] alertmanager_client: # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -ruler.alertmanager-client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -ruler.alertmanager-client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. If # not set, the host's root CA certificates are used. # CLI flag: -ruler.alertmanager-client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -ruler.alertmanager-client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -ruler.alertmanager-client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # HTTP Basic authentication username. It overrides the username set in the URL # (if any). # CLI flag: -ruler.alertmanager-client.basic-auth-username [basic_auth_username: \u003cstring\u003e | default = \"\"] # HTTP Basic authentication password. It overrides the password set in the URL # (if any). # CLI flag: -ruler.alertmanager-client.basic-auth-password [basic_auth_password: \u003cstring\u003e | default = \"\"] # Max time to tolerate outage for restoring \"for\" state of alert. # CLI flag: -ruler.for-outage-tolerance [for_outage_tolerance: \u003cduration\u003e | default = 1h] # Minimum duration between alert and restored \"for\" state. This is maintained # only for alerts with configured \"for\" time greater than grace period. # CLI flag: -ruler.for-grace-period [for_grace_period: \u003cduration\u003e | default = 10m] # Minimum amount of time to wait before resending an alert to Alertmanager. # CLI flag: -ruler.resend-delay [resend_delay: \u003cduration\u003e | default = 1m] # If enabled, rules from a single rule group can be evaluated concurrently if # there is no dependency between each other. Max concurrency for each rule group # is controlled via ruler.max-concurrent-evals flag. # CLI flag: -ruler.concurrent-evals-enabled [concurrent_evals_enabled: \u003cboolean\u003e | default = false] # Max concurrency for a single rule group to evaluate independent rules. # CLI flag: -ruler.max-concurrent-evals [max_concurrent_evals: \u003cint\u003e | default = 1] # Distribute rule evaluation using ring backend # CLI flag: -ruler.enable-sharding [enable_sharding: \u003cboolean\u003e | default = false] # The sharding strategy to use. Supported values are: default, shuffle-sharding. # CLI flag: -ruler.sharding-strategy [sharding_strategy: \u003cstring\u003e | default = \"default\"] # Time to spend searching for a pending ruler when shutting down. # CLI flag: -ruler.search-pending-for [search_pending_for: \u003cduration\u003e | default = 5m] ring: kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -ruler.ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -ruler.ring.prefix [prefix: \u003cstring\u003e | default = \"rulers/\"] dynamodb: # Region to access dynamodb. # CLI flag: -ruler.ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -ruler.ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -ruler.ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -ruler.ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -ruler.ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: ruler.ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: ruler.ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -ruler.ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -ruler.ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -ruler.ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -ruler.ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -ruler.ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 5s] # The heartbeat timeout after which rulers are considered unhealthy within the # ring. 0 = never (timeout disabled). # CLI flag: -ruler.ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # EXPERIMENTAL: The replication factor to use when loading rule groups for API # HA. # CLI flag: -ruler.ring.replication-factor [replication_factor: \u003cint\u003e | default = 1] # EXPERIMENTAL: True to enable zone-awareness and load rule groups across # different availability zones for API HA. # CLI flag: -ruler.ring.zone-awareness-enabled [zone_awareness_enabled: \u003cboolean\u003e | default = false] # EXPERIMENTAL: File path where tokens are stored. If empty, tokens are not # stored at shutdown and restored at startup. # CLI flag: -ruler.ring.tokens-file-path [tokens_file_path: \u003cstring\u003e | default = \"\"] # Name of network interface to read address from. # CLI flag: -ruler.ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # Number of tokens for each ruler. # CLI flag: -ruler.ring.num-tokens [num_tokens: \u003cint\u003e | default = 128] # The sleep seconds when ruler is shutting down. Need to be close to or larger # than KV Store information propagation delay # CLI flag: -ruler.ring.final-sleep [final_sleep: \u003cduration\u003e | default = 0s] # Keep instance in the ring on shut down. # CLI flag: -ruler.ring.keep-instance-in-the-ring-on-shutdown [keep_instance_in_the_ring_on_shutdown: \u003cboolean\u003e | default = false] # Period with which to attempt to flush rule groups. # CLI flag: -ruler.flush-period [flush_period: \u003cduration\u003e | default = 1m] # Enable the ruler api # CLI flag: -experimental.ruler.enable-api [enable_api: \u003cboolean\u003e | default = false] # EXPERIMENTAL: Remove duplicate rules in the prometheus rules and alerts API # response. If there are duplicate rules the rule with the latest evaluation # timestamp will be kept. # CLI flag: -experimental.ruler.api-deduplicate-rules [api_deduplicate_rules: \u003cboolean\u003e | default = false] # Comma separated list of tenants whose rules this ruler can evaluate. If # specified, only these tenants will be handled by ruler, otherwise this ruler # can process rules from all tenants. Subject to sharding. # CLI flag: -ruler.enabled-tenants [enabled_tenants: \u003cstring\u003e | default = \"\"] # Comma separated list of tenants whose rules this ruler cannot evaluate. If # specified, a ruler that would normally pick the specified tenant(s) for # processing will ignore them instead. Subject to sharding. # CLI flag: -ruler.disabled-tenants [disabled_tenants: \u003cstring\u003e | default = \"\"] # Report query statistics for ruler queries to complete as a per user metric and # as an info level log message. # CLI flag: -ruler.query-stats-enabled [query_stats_enabled: \u003cboolean\u003e | default = false] # Disable the rule_group label on exported metrics # CLI flag: -ruler.disable-rule-group-label [disable_rule_group_label: \u003cboolean\u003e | default = false] # Enable high availability # CLI flag: -ruler.enable-ha-evaluation [enable_ha_evaluation: \u003cboolean\u003e | default = false] # Timeout duration for non-primary rulers during liveness checks. If the check # times out, the non-primary ruler will evaluate the rule group. Applicable when # ruler.enable-ha-evaluation is true. # CLI flag: -ruler.liveness-check-timeout [liveness_check_timeout: \u003cduration\u003e | default = 1s] ruler_storage_config The ruler_storage_config configures the Cortex ruler storage backend.\n# Backend storage to use. Supported backends are: s3, gcs, azure, swift, # filesystem, configdb, local. # CLI flag: -ruler-storage.backend [backend: \u003cstring\u003e | default = \"s3\"] s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an # S3-compatible service in hostname:port format. # CLI flag: -ruler-storage.s3.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # S3 region. If unset, the client will issue a S3 GetBucketLocation API call # to autodetect it. # CLI flag: -ruler-storage.s3.region [region: \u003cstring\u003e | default = \"\"] # S3 bucket name # CLI flag: -ruler-storage.s3.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # S3 secret access key # CLI flag: -ruler-storage.s3.secret-access-key [secret_access_key: \u003cstring\u003e | default = \"\"] # S3 access key ID # CLI flag: -ruler-storage.s3.access-key-id [access_key_id: \u003cstring\u003e | default = \"\"] # If enabled, use http:// for the S3 endpoint instead of https://. This could # be useful in local dev/test environments while using an S3-compatible # backend storage, like Minio. # CLI flag: -ruler-storage.s3.insecure [insecure: \u003cboolean\u003e | default = false] # The signature version to use for authenticating against S3. Supported values # are: v4, v2. # CLI flag: -ruler-storage.s3.signature-version [signature_version: \u003cstring\u003e | default = \"v4\"] # The s3 bucket lookup style. Supported values are: auto, virtual-hosted, # path. # CLI flag: -ruler-storage.s3.bucket-lookup-type [bucket_lookup_type: \u003cstring\u003e | default = \"auto\"] # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum # algorithm to verify the provided digest. If false, use CRC32C algorithm # instead. # CLI flag: -ruler-storage.s3.send-content-md5 [send_content_md5: \u003cboolean\u003e | default = true] # The list api version. Supported values are: v1, v2, and ''. # CLI flag: -ruler-storage.s3.list-objects-version [list_objects_version: \u003cstring\u003e | default = \"\"] # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: ruler-storage [sse: \u003cs3_sse_config\u003e] http: # The time an idle connection will remain idle before closing. # CLI flag: -ruler-storage.s3.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -ruler-storage.s3.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -ruler-storage.s3.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -ruler-storage.s3.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -ruler-storage.s3.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -ruler-storage.s3.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -ruler-storage.s3.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -ruler-storage.s3.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] gcs: # GCS bucket name # CLI flag: -ruler-storage.gcs.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # JSON representing either a Google Developers Console client_credentials.json # file or a Google Developers service account key file. If empty, fallback to # Google default logic. # CLI flag: -ruler-storage.gcs.service-account [service_account: \u003cstring\u003e | default = \"\"] azure: # Azure storage account name # CLI flag: -ruler-storage.azure.account-name [account_name: \u003cstring\u003e | default = \"\"] # Azure storage account key # CLI flag: -ruler-storage.azure.account-key [account_key: \u003cstring\u003e | default = \"\"] # The values of `account-name` and `endpoint-suffix` values will not be # ignored if `connection-string` is set. Use this method over `account-key` if # you need to authenticate via a SAS token or if you use the Azurite emulator. # CLI flag: -ruler-storage.azure.connection-string [connection_string: \u003cstring\u003e | default = \"\"] # Azure storage container name # CLI flag: -ruler-storage.azure.container-name [container_name: \u003cstring\u003e | default = \"\"] # Azure storage endpoint suffix without schema. The account name will be # prefixed to this value to create the FQDN # CLI flag: -ruler-storage.azure.endpoint-suffix [endpoint_suffix: \u003cstring\u003e | default = \"\"] # Number of retries for recoverable errors # CLI flag: -ruler-storage.azure.max-retries [max_retries: \u003cint\u003e | default = 20] # Deprecated: Azure storage MSI resource. It will be set automatically by # Azure SDK. # CLI flag: -ruler-storage.azure.msi-resource [msi_resource: \u003cstring\u003e | default = \"\"] # Azure storage MSI resource managed identity client Id. If not supplied # default Azure credential will be used. Set it to empty if you need to # authenticate via Azure Workload Identity. # CLI flag: -ruler-storage.azure.user-assigned-id [user_assigned_id: \u003cstring\u003e | default = \"\"] http: # The time an idle connection will remain idle before closing. # CLI flag: -ruler-storage.azure.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -ruler-storage.azure.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -ruler-storage.azure.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -ruler-storage.azure.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -ruler-storage.azure.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -ruler-storage.azure.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -ruler-storage.azure.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -ruler-storage.azure.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] swift: # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -ruler-storage.swift.auth-version [auth_version: \u003cint\u003e | default = 0] # OpenStack Swift authentication URL # CLI flag: -ruler-storage.swift.auth-url [auth_url: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential ID. # CLI flag: -ruler-storage.swift.application-credential-id [application_credential_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential name. # CLI flag: -ruler-storage.swift.application-credential-name [application_credential_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential secret. # CLI flag: -ruler-storage.swift.application-credential-secret [application_credential_secret: \u003cstring\u003e | default = \"\"] # OpenStack Swift username. # CLI flag: -ruler-storage.swift.username [username: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -ruler-storage.swift.user-domain-name [user_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -ruler-storage.swift.user-domain-id [user_domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user ID. # CLI flag: -ruler-storage.swift.user-id [user_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift API key. # CLI flag: -ruler-storage.swift.password [password: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -ruler-storage.swift.domain-id [domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -ruler-storage.swift.domain-name [domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift project ID (v2,v3 auth only). # CLI flag: -ruler-storage.swift.project-id [project_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift project name (v2,v3 auth only). # CLI flag: -ruler-storage.swift.project-name [project_name: \u003cstring\u003e | default = \"\"] # ID of the OpenStack Swift project's domain (v3 auth only), only needed if it # differs the from user domain. # CLI flag: -ruler-storage.swift.project-domain-id [project_domain_id: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift project's domain (v3 auth only), only needed if # it differs from the user domain. # CLI flag: -ruler-storage.swift.project-domain-name [project_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift Region to use (v2,v3 auth only). # CLI flag: -ruler-storage.swift.region-name [region_name: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift container to put chunks in. # CLI flag: -ruler-storage.swift.container-name [container_name: \u003cstring\u003e | default = \"\"] # Max retries on requests error. # CLI flag: -ruler-storage.swift.max-retries [max_retries: \u003cint\u003e | default = 3] # Time after which a connection attempt is aborted. # CLI flag: -ruler-storage.swift.connect-timeout [connect_timeout: \u003cduration\u003e | default = 10s] # Time after which an idle request is aborted. The timeout watchdog is reset # each time some data is received, so the timeout triggers after X time no # data is received on a request. # CLI flag: -ruler-storage.swift.request-timeout [request_timeout: \u003cduration\u003e | default = 5s] filesystem: # Local filesystem storage directory. # CLI flag: -ruler-storage.filesystem.dir [dir: \u003cstring\u003e | default = \"\"] # The configstore_config configures the config database storing rules and # alerts, and is used by the Cortex alertmanager. # The CLI flags prefix for this block config is: ruler-storage [configdb: \u003cconfigstore_config\u003e] local: # Directory to scan for rules # CLI flag: -ruler-storage.local.directory [directory: \u003cstring\u003e | default = \"\"] runtime_configuration_storage_config The runtime_configuration_storage_config configures the storage backend for the runtime configuration file.\n# How often to check runtime config file. # CLI flag: -runtime-config.reload-period [period: \u003cduration\u003e | default = 10s] # File with the configuration that can be updated in runtime. # CLI flag: -runtime-config.file [file: \u003cstring\u003e | default = \"\"] # Backend storage to use. Supported backends are: s3, gcs, azure, swift, # filesystem. # CLI flag: -runtime-config.backend [backend: \u003cstring\u003e | default = \"filesystem\"] s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an # S3-compatible service in hostname:port format. # CLI flag: -runtime-config.s3.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # S3 region. If unset, the client will issue a S3 GetBucketLocation API call # to autodetect it. # CLI flag: -runtime-config.s3.region [region: \u003cstring\u003e | default = \"\"] # S3 bucket name # CLI flag: -runtime-config.s3.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # S3 secret access key # CLI flag: -runtime-config.s3.secret-access-key [secret_access_key: \u003cstring\u003e | default = \"\"] # S3 access key ID # CLI flag: -runtime-config.s3.access-key-id [access_key_id: \u003cstring\u003e | default = \"\"] # If enabled, use http:// for the S3 endpoint instead of https://. This could # be useful in local dev/test environments while using an S3-compatible # backend storage, like Minio. # CLI flag: -runtime-config.s3.insecure [insecure: \u003cboolean\u003e | default = false] # The signature version to use for authenticating against S3. Supported values # are: v4, v2. # CLI flag: -runtime-config.s3.signature-version [signature_version: \u003cstring\u003e | default = \"v4\"] # The s3 bucket lookup style. Supported values are: auto, virtual-hosted, # path. # CLI flag: -runtime-config.s3.bucket-lookup-type [bucket_lookup_type: \u003cstring\u003e | default = \"auto\"] # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum # algorithm to verify the provided digest. If false, use CRC32C algorithm # instead. # CLI flag: -runtime-config.s3.send-content-md5 [send_content_md5: \u003cboolean\u003e | default = true] # The list api version. Supported values are: v1, v2, and ''. # CLI flag: -runtime-config.s3.list-objects-version [list_objects_version: \u003cstring\u003e | default = \"\"] # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: runtime-config [sse: \u003cs3_sse_config\u003e] http: # The time an idle connection will remain idle before closing. # CLI flag: -runtime-config.s3.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -runtime-config.s3.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -runtime-config.s3.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -runtime-config.s3.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -runtime-config.s3.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -runtime-config.s3.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -runtime-config.s3.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -runtime-config.s3.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] gcs: # GCS bucket name # CLI flag: -runtime-config.gcs.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # JSON representing either a Google Developers Console client_credentials.json # file or a Google Developers service account key file. If empty, fallback to # Google default logic. # CLI flag: -runtime-config.gcs.service-account [service_account: \u003cstring\u003e | default = \"\"] azure: # Azure storage account name # CLI flag: -runtime-config.azure.account-name [account_name: \u003cstring\u003e | default = \"\"] # Azure storage account key # CLI flag: -runtime-config.azure.account-key [account_key: \u003cstring\u003e | default = \"\"] # The values of `account-name` and `endpoint-suffix` values will not be # ignored if `connection-string` is set. Use this method over `account-key` if # you need to authenticate via a SAS token or if you use the Azurite emulator. # CLI flag: -runtime-config.azure.connection-string [connection_string: \u003cstring\u003e | default = \"\"] # Azure storage container name # CLI flag: -runtime-config.azure.container-name [container_name: \u003cstring\u003e | default = \"\"] # Azure storage endpoint suffix without schema. The account name will be # prefixed to this value to create the FQDN # CLI flag: -runtime-config.azure.endpoint-suffix [endpoint_suffix: \u003cstring\u003e | default = \"\"] # Number of retries for recoverable errors # CLI flag: -runtime-config.azure.max-retries [max_retries: \u003cint\u003e | default = 20] # Deprecated: Azure storage MSI resource. It will be set automatically by # Azure SDK. # CLI flag: -runtime-config.azure.msi-resource [msi_resource: \u003cstring\u003e | default = \"\"] # Azure storage MSI resource managed identity client Id. If not supplied # default Azure credential will be used. Set it to empty if you need to # authenticate via Azure Workload Identity. # CLI flag: -runtime-config.azure.user-assigned-id [user_assigned_id: \u003cstring\u003e | default = \"\"] http: # The time an idle connection will remain idle before closing. # CLI flag: -runtime-config.azure.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -runtime-config.azure.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -runtime-config.azure.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -runtime-config.azure.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully writing # the request headers if the request has an Expect header. 0 to send the # request body immediately. # CLI flag: -runtime-config.azure.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 means # no limit. # CLI flag: -runtime-config.azure.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, a # built-in default value is used. # CLI flag: -runtime-config.azure.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -runtime-config.azure.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] swift: # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -runtime-config.swift.auth-version [auth_version: \u003cint\u003e | default = 0] # OpenStack Swift authentication URL # CLI flag: -runtime-config.swift.auth-url [auth_url: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential ID. # CLI flag: -runtime-config.swift.application-credential-id [application_credential_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential name. # CLI flag: -runtime-config.swift.application-credential-name [application_credential_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential secret. # CLI flag: -runtime-config.swift.application-credential-secret [application_credential_secret: \u003cstring\u003e | default = \"\"] # OpenStack Swift username. # CLI flag: -runtime-config.swift.username [username: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -runtime-config.swift.user-domain-name [user_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -runtime-config.swift.user-domain-id [user_domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user ID. # CLI flag: -runtime-config.swift.user-id [user_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift API key. # CLI flag: -runtime-config.swift.password [password: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -runtime-config.swift.domain-id [domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -runtime-config.swift.domain-name [domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift project ID (v2,v3 auth only). # CLI flag: -runtime-config.swift.project-id [project_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift project name (v2,v3 auth only). # CLI flag: -runtime-config.swift.project-name [project_name: \u003cstring\u003e | default = \"\"] # ID of the OpenStack Swift project's domain (v3 auth only), only needed if it # differs the from user domain. # CLI flag: -runtime-config.swift.project-domain-id [project_domain_id: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift project's domain (v3 auth only), only needed if # it differs from the user domain. # CLI flag: -runtime-config.swift.project-domain-name [project_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift Region to use (v2,v3 auth only). # CLI flag: -runtime-config.swift.region-name [region_name: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift container to put chunks in. # CLI flag: -runtime-config.swift.container-name [container_name: \u003cstring\u003e | default = \"\"] # Max retries on requests error. # CLI flag: -runtime-config.swift.max-retries [max_retries: \u003cint\u003e | default = 3] # Time after which a connection attempt is aborted. # CLI flag: -runtime-config.swift.connect-timeout [connect_timeout: \u003cduration\u003e | default = 10s] # Time after which an idle request is aborted. The timeout watchdog is reset # each time some data is received, so the timeout triggers after X time no # data is received on a request. # CLI flag: -runtime-config.swift.request-timeout [request_timeout: \u003cduration\u003e | default = 5s] filesystem: # Local filesystem storage directory. # CLI flag: -runtime-config.filesystem.dir [dir: \u003cstring\u003e | default = \"\"] s3_sse_config The s3_sse_config configures the S3 server-side encryption. The supported CLI flags \u003cprefix\u003e used to reference this config block are:\nalertmanager-storage blocks-storage ruler-storage runtime-config # Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3. # CLI flag: -\u003cprefix\u003e.s3.sse.type [type: \u003cstring\u003e | default = \"\"] # KMS Key ID used to encrypt objects in S3 # CLI flag: -\u003cprefix\u003e.s3.sse.kms-key-id [kms_key_id: \u003cstring\u003e | default = \"\"] # KMS Encryption Context used for object encryption. It expects JSON formatted # string. # CLI flag: -\u003cprefix\u003e.s3.sse.kms-encryption-context [kms_encryption_context: \u003cstring\u003e | default = \"\"] server_config The server_config configures the HTTP and gRPC server of the launched service(s).\n# HTTP server listen network, default tcp # CLI flag: -server.http-listen-network [http_listen_network: \u003cstring\u003e | default = \"tcp\"] # HTTP server listen address. # CLI flag: -server.http-listen-address [http_listen_address: \u003cstring\u003e | default = \"\"] # HTTP server listen port. # CLI flag: -server.http-listen-port [http_listen_port: \u003cint\u003e | default = 80] # Maximum number of simultaneous http connections, \u003c=0 to disable # CLI flag: -server.http-conn-limit [http_listen_conn_limit: \u003cint\u003e | default = 0] # gRPC server listen network # CLI flag: -server.grpc-listen-network [grpc_listen_network: \u003cstring\u003e | default = \"tcp\"] # gRPC server listen address. # CLI flag: -server.grpc-listen-address [grpc_listen_address: \u003cstring\u003e | default = \"\"] # gRPC server listen port. # CLI flag: -server.grpc-listen-port [grpc_listen_port: \u003cint\u003e | default = 9095] # Maximum number of simultaneous grpc connections, \u003c=0 to disable # CLI flag: -server.grpc-conn-limit [grpc_listen_conn_limit: \u003cint\u003e | default = 0] # Comma-separated list of cipher suites to use. If blank, the default Go cipher # suites is used. # CLI flag: -server.tls-cipher-suites [tls_cipher_suites: \u003cstring\u003e | default = \"\"] # Minimum TLS version to use. Allowed values: VersionTLS10, VersionTLS11, # VersionTLS12, VersionTLS13. If blank, the Go TLS minimum version is used. # CLI flag: -server.tls-min-version [tls_min_version: \u003cstring\u003e | default = \"\"] http_tls_config: # HTTP server cert path. # CLI flag: -server.http-tls-cert-path [cert_file: \u003cstring\u003e | default = \"\"] # HTTP server key path. # CLI flag: -server.http-tls-key-path [key_file: \u003cstring\u003e | default = \"\"] # HTTP TLS Client Auth type. # CLI flag: -server.http-tls-client-auth [client_auth_type: \u003cstring\u003e | default = \"\"] # HTTP TLS Client CA path. # CLI flag: -server.http-tls-ca-path [client_ca_file: \u003cstring\u003e | default = \"\"] grpc_tls_config: # GRPC TLS server cert path. # CLI flag: -server.grpc-tls-cert-path [cert_file: \u003cstring\u003e | default = \"\"] # GRPC TLS server key path. # CLI flag: -server.grpc-tls-key-path [key_file: \u003cstring\u003e | default = \"\"] # GRPC TLS Client Auth type. # CLI flag: -server.grpc-tls-client-auth [client_auth_type: \u003cstring\u003e | default = \"\"] # GRPC TLS Client CA path. # CLI flag: -server.grpc-tls-ca-path [client_ca_file: \u003cstring\u003e | default = \"\"] # Register the intrumentation handlers (/metrics etc). # CLI flag: -server.register-instrumentation [register_instrumentation: \u003cboolean\u003e | default = true] # Timeout for graceful shutdowns # CLI flag: -server.graceful-shutdown-timeout [graceful_shutdown_timeout: \u003cduration\u003e | default = 30s] # Read timeout for HTTP server # CLI flag: -server.http-read-timeout [http_server_read_timeout: \u003cduration\u003e | default = 30s] # Write timeout for HTTP server # CLI flag: -server.http-write-timeout [http_server_write_timeout: \u003cduration\u003e | default = 30s] # Idle timeout for HTTP server # CLI flag: -server.http-idle-timeout [http_server_idle_timeout: \u003cduration\u003e | default = 2m] # Limit on the size of a gRPC message this server can receive (bytes). # CLI flag: -server.grpc-max-recv-msg-size-bytes [grpc_server_max_recv_msg_size: \u003cint\u003e | default = 4194304] # Limit on the size of a gRPC message this server can send (bytes). # CLI flag: -server.grpc-max-send-msg-size-bytes [grpc_server_max_send_msg_size: \u003cint\u003e | default = 4194304] # Limit on the number of concurrent streams for gRPC calls (0 = unlimited) # CLI flag: -server.grpc-max-concurrent-streams [grpc_server_max_concurrent_streams: \u003cint\u003e | default = 100] # Number of worker goroutines that should be used to process incoming # streams.Setting this 0 (default) will disable workers and spawn a new # goroutine for each stream. # CLI flag: -server.grpc_server-num-stream-workers [grpc_server_num_stream_workers: \u003cint\u003e | default = 0] # The duration after which an idle connection should be closed. Default: # infinity # CLI flag: -server.grpc.keepalive.max-connection-idle [grpc_server_max_connection_idle: \u003cduration\u003e | default = 2562047h47m16.854775807s] # The duration for the maximum amount of time a connection may exist before it # will be closed. Default: infinity # CLI flag: -server.grpc.keepalive.max-connection-age [grpc_server_max_connection_age: \u003cduration\u003e | default = 2562047h47m16.854775807s] # An additive period after max-connection-age after which the connection will be # forcibly closed. Default: infinity # CLI flag: -server.grpc.keepalive.max-connection-age-grace [grpc_server_max_connection_age_grace: \u003cduration\u003e | default = 2562047h47m16.854775807s] # Duration after which a keepalive probe is sent in case of no activity over the # connection., Default: 2h # CLI flag: -server.grpc.keepalive.time [grpc_server_keepalive_time: \u003cduration\u003e | default = 2h] # After having pinged for keepalive check, the duration after which an idle # connection should be closed, Default: 20s # CLI flag: -server.grpc.keepalive.timeout [grpc_server_keepalive_timeout: \u003cduration\u003e | default = 20s] # Minimum amount of time a client should wait before sending a keepalive ping. # If client sends keepalive ping more often, server will send GOAWAY and close # the connection. # CLI flag: -server.grpc.keepalive.min-time-between-pings [grpc_server_min_time_between_pings: \u003cduration\u003e | default = 10s] # If true, server allows keepalive pings even when there are no active # streams(RPCs). If false, and client sends ping when there are no active # streams, server will send GOAWAY and close the connection. # CLI flag: -server.grpc.keepalive.ping-without-stream-allowed [grpc_server_ping_without_stream_allowed: \u003cboolean\u003e | default = true] # Output log messages in the given format. Valid formats: [logfmt, json] # CLI flag: -log.format [log_format: \u003cstring\u003e | default = \"logfmt\"] # Only log messages with the given severity or above. Valid levels: [debug, # info, warn, error] # CLI flag: -log.level [log_level: \u003cstring\u003e | default = \"info\"] # Optionally log the source IPs. # CLI flag: -server.log-source-ips-enabled [log_source_ips_enabled: \u003cboolean\u003e | default = false] # Header field storing the source IPs. Only used if # server.log-source-ips-enabled is true. If not set the default Forwarded, # X-Real-IP and X-Forwarded-For headers are used # CLI flag: -server.log-source-ips-header [log_source_ips_header: \u003cstring\u003e | default = \"\"] # Regex for matching the source IPs. Only used if server.log-source-ips-enabled # is true. If not set the default Forwarded, X-Real-IP and X-Forwarded-For # headers are used # CLI flag: -server.log-source-ips-regex [log_source_ips_regex: \u003cstring\u003e | default = \"\"] # Optionally log request headers. # CLI flag: -server.log-request-headers [log_request_headers: \u003cboolean\u003e | default = false] # Optionally log requests at info level instead of debug level. Applies to # request headers as well if server.log-request-headers is enabled. # CLI flag: -server.log-request-at-info-level-enabled [log_request_at_info_level_enabled: \u003cboolean\u003e | default = false] # Comma separated list of headers to exclude from loggin. Only used if # server.log-request-headers is true. # CLI flag: -server.log-request-headers-exclude-list [log_request_exclude_headers_list: \u003cstring\u003e | default = \"\"] # Base path to serve all API routes from (e.g. /v1/) # CLI flag: -server.path-prefix [http_path_prefix: \u003cstring\u003e | default = \"\"] storage_config The storage_config configures the storage type Cortex uses.\n# The storage engine to use: blocks is the only supported option today. # CLI flag: -store.engine [engine: \u003cstring\u003e | default = \"blocks\"] store_gateway_config The store_gateway_config configures the store-gateway service used by the blocks storage.\n# Shard blocks across multiple store gateway instances. This option needs be set # both on the store-gateway and querier when running in microservices mode. # CLI flag: -store-gateway.sharding-enabled [sharding_enabled: \u003cboolean\u003e | default = false] # The hash ring configuration. This option is required only if blocks sharding # is enabled. sharding_ring: # The key-value store used to share the hash ring across multiple instances. # This option needs be set both on the store-gateway and querier when running # in microservices mode. kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -store-gateway.sharding-ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -store-gateway.sharding-ring.prefix [prefix: \u003cstring\u003e | default = \"collectors/\"] dynamodb: # Region to access dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -store-gateway.sharding-ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: store-gateway.sharding-ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: store-gateway.sharding-ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -store-gateway.sharding-ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -store-gateway.sharding-ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -store-gateway.sharding-ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -store-gateway.sharding-ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -store-gateway.sharding-ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 15s] # The heartbeat timeout after which store gateways are considered unhealthy # within the ring. 0 = never (timeout disabled). This option needs be set both # on the store-gateway and querier when running in microservices mode. # CLI flag: -store-gateway.sharding-ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # The replication factor to use when sharding blocks. This option needs be set # both on the store-gateway and querier when running in microservices mode. # CLI flag: -store-gateway.sharding-ring.replication-factor [replication_factor: \u003cint\u003e | default = 3] # File path where tokens are stored. If empty, tokens are not stored at # shutdown and restored at startup. # CLI flag: -store-gateway.sharding-ring.tokens-file-path [tokens_file_path: \u003cstring\u003e | default = \"\"] # True to enable zone-awareness and replicate blocks across different # availability zones. # CLI flag: -store-gateway.sharding-ring.zone-awareness-enabled [zone_awareness_enabled: \u003cboolean\u003e | default = false] # True to keep the store gateway instance in the ring when it shuts down. The # instance will then be auto-forgotten from the ring after # 10*heartbeat_timeout. # CLI flag: -store-gateway.sharding-ring.keep-instance-in-the-ring-on-shutdown [keep_instance_in_the_ring_on_shutdown: \u003cboolean\u003e | default = false] # Minimum time to wait for ring stability at startup. 0 to disable. # CLI flag: -store-gateway.sharding-ring.wait-stability-min-duration [wait_stability_min_duration: \u003cduration\u003e | default = 1m] # Maximum time to wait for ring stability at startup. If the store-gateway # ring keeps changing after this period of time, the store-gateway will start # anyway. # CLI flag: -store-gateway.sharding-ring.wait-stability-max-duration [wait_stability_max_duration: \u003cduration\u003e | default = 5m] # Timeout for waiting on store-gateway to become desired state in the ring. # CLI flag: -store-gateway.sharding-ring.wait-instance-state-timeout [wait_instance_state_timeout: \u003cduration\u003e | default = 10m] # The sleep seconds when store-gateway is shutting down. Need to be close to # or larger than KV Store information propagation delay # CLI flag: -store-gateway.sharding-ring.final-sleep [final_sleep: \u003cduration\u003e | default = 0s] # Name of network interface to read address from. # CLI flag: -store-gateway.sharding-ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # The availability zone where this instance is running. Required if # zone-awareness is enabled. # CLI flag: -store-gateway.sharding-ring.instance-availability-zone [instance_availability_zone: \u003cstring\u003e | default = \"\"] # The sharding strategy to use. Supported values are: default, shuffle-sharding. # CLI flag: -store-gateway.sharding-strategy [sharding_strategy: \u003cstring\u003e | default = \"default\"] # Comma separated list of tenants whose store metrics this storegateway can # process. If specified, only these tenants will be handled by storegateway, # otherwise this storegateway will be enabled for all the tenants in the # store-gateway cluster. # CLI flag: -store-gateway.enabled-tenants [enabled_tenants: \u003cstring\u003e | default = \"\"] # Comma separated list of tenants whose store metrics this storegateway cannot # process. If specified, a storegateway that would normally pick the specified # tenant(s) for processing will ignore them instead. # CLI flag: -store-gateway.disabled-tenants [disabled_tenants: \u003cstring\u003e | default = \"\"] hedged_request: # If true, hedged requests are applied to object store calls. It can help with # reducing tail latency. # CLI flag: -store-gateway.hedged-request.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of hedged requests allowed for each initial request. A high # number can reduce latency but increase internal calls. # CLI flag: -store-gateway.hedged-request.max-requests [max_requests: \u003cint\u003e | default = 3] # It is used to calculate a latency threshold to trigger hedged requests. For # example, additional requests are triggered when the initial request response # time exceeds the 90th percentile. # CLI flag: -store-gateway.hedged-request.quantile [quantile: \u003cfloat\u003e | default = 0.9] tracing_config The tracing_config configures backends cortex uses.\n# Tracing type. OTEL and JAEGER are currently supported. For jaeger # `JAEGER_AGENT_HOST` environment variable should also be set. See: # https://cortexmetrics.io/docs/guides/tracing . # CLI flag: -tracing.type [type: \u003cstring\u003e | default = \"jaeger\"] otel: # otl collector endpoint that the driver will use to send spans. # CLI flag: -tracing.otel.otlp-endpoint [otlp_endpoint: \u003cstring\u003e | default = \"\"] # enhance/modify traces/propagators for specific exporter. If empty, OTEL # defaults will apply. Supported values are: `awsxray.` # CLI flag: -tracing.otel.exporter-type [exporter_type: \u003cstring\u003e | default = \"\"] # Fraction of traces to be sampled. Fractions \u003e= 1 means sampling if off and # everything is traced. # CLI flag: -tracing.otel.sample-ratio [sample_ratio: \u003cfloat\u003e | default = 0.001] # If enabled, use round_robin gRPC load balancing policy. By default, use # pick_first policy. For more details, please refer to # https://github.com/grpc/grpc/blob/master/doc/load-balancing.md#load-balancing-policies. # CLI flag: -tracing.otel.round-robin [round_robin: \u003cboolean\u003e | default = false] # Enable TLS in the GRPC client. This flag needs to be enabled when any other # TLS flag is set. If set to false, insecure connection to gRPC server will be # used. # CLI flag: -tracing.otel.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] tls: # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -tracing.otel.tls.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -tracing.otel.tls.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. # If not set, the host's root CA certificates are used. # CLI flag: -tracing.otel.tls.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -tracing.otel.tls.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -tracing.otel.tls.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] LimitsPerLabelSet limits: # The maximum number of active series per LabelSet, across the cluster before # replication. Setting the value 0 will enable the monitoring (metrics) but # would not enforce any limits. [max_series: \u003cint\u003e | default = ] # LabelSet which the limit should be applied. [label_set: \u003cmap of string (labelName) to string (labelValue)\u003e | default = []] PriorityDef # Priority level. Must be a unique value. [priority: \u003cint\u003e | default = 0] # Number of reserved queriers to handle priorities higher or equal to the # priority level. Value between 0 and 1 will be used as a percentage. [reserved_queriers: \u003cfloat\u003e | default = 0] # List of query_attributes to match and assign priority to queries. A query is # assigned to this priority if it matches any query_attribute in this list. Each # query_attribute has several properties (e.g., regex, time_window, user_agent), # and all specified properties must match for a query_attribute to be considered # a match. Only the specified properties are checked, and an AND operator is # applied to them. [query_attributes: \u003clist of QueryAttribute\u003e | default = []] QueryAttribute # API type for the query. Should be one of the query, query_range, series, # labels, label_values. If not set, it won't be checked. [api_type: \u003cstring\u003e | default = \"\"] # Regex that the query string (or at least one of the matchers in metadata # query) should match. If not set, it won't be checked. [regex: \u003cstring\u003e | default = \"\"] # Overall data select time window (including range selectors, modifiers and # lookback delta) that the query should be within. If not set, it won't be # checked. time_window: # Start of the data select time window (including range selectors, modifiers # and lookback delta) that the query should be within. If set to 0, it won't # be checked. [start: \u003cint\u003e | default = 0] # End of the data select time window (including range selectors, modifiers and # lookback delta) that the query should be within. If set to 0, it won't be # checked. [end: \u003cint\u003e | default = 0] # Query time range should be within this limit to match. Depending on where it # was used, in most of the use-cases, either min or max value will be used. If # not set, it won't be checked. time_range_limit: # This will be duration (12h, 1d, 15d etc.). Query time range should be above # or equal to this value to match. Ex: if this value is 20d, then queries # whose range is bigger than or equal to 20d will match. If set to 0, it won't # be checked. [min: \u003cint\u003e | default = 0] # This will be duration (12h, 1d, 15d etc.). Query time range should be below # or equal to this value to match. Ex: if this value is 24h, then queries # whose range is smaller than or equal to 24h will match.If set to 0, it won't # be checked. [max: \u003cint\u003e | default = 0] # If query step provided should be within this limit to match. If not set, it # won't be checked. This property only applied to range queries and ignored for # other types of queries. query_step_limit: # Query step should be above or equal to this value to match. If set to 0, it # won't be checked. [min: \u003cint\u003e | default = 0] # Query step should be below or equal to this value to match. If set to 0, it # won't be checked. [max: \u003cint\u003e | default = 0] # Regex that User-Agent header of the request should match. If not set, it won't # be checked. [user_agent_regex: \u003cstring\u003e | default = \"\"] # Grafana includes X-Dashboard-Uid header in query requests. If this field is # provided then X-Dashboard-Uid header of request should match this value. If # not set, it won't be checked. This property won't be applied to metadata # queries. [dashboard_uid: \u003cstring\u003e | default = \"\"] # Grafana includes X-Panel-Id header in query requests. If this field is # provided then X-Panel-Id header of request should match this value. If not # set, it won't be checked. This property won't be applied to metadata queries. [panel_id: \u003cstring\u003e | default = \"\"] DisabledRuleGroup # namespace in which the rule group belongs [namespace: \u003cstring\u003e | default = \"\"] # name of the rule group [name: \u003cstring\u003e | default = \"\"] ","categories":"","description":"","excerpt":" Cortex can be configured using a YAML file - specified using the …","ref":"/docs/configuration/configuration-file/","tags":"","title":"Configuration file"},{"body":" Author: Christian Simon Date: October 2020 Status: Accepted Overview This document aims to describe how to implement the ability to allow queries to cover data from more than a single Cortex tenant.\nReasoning Adopting a tenancy model within an organization with each tenant representing a department comes with the disadvantage that it will prevent queries from spanning multiple departments. This proposal tries to overcome those limitations.\nAlternatives considered Aggregation in PromQL API clients In theory PromQL API clients could be aggregating/correlating query results from multiple tenants. For example Grafana could be used with multiple data sources and a cross tenant query could be achieved through using Transformations.\nAs this approach comes with the following disadvantages, it was not considered further:\nEvery PromQL API client needs to support the aggregation from various sources.\nQueries that are written in PromQL can’t be used without extra work across tenants.\nMulti tenant aggregation in the query frontends Another approach to multi tenant query federation could be achieved by aggregation of partial query results within the query frontend. For this a query needs to be split into sub queries per tenant and afterwards the partial results need reduced into the final result.\nThe astmapper package goes down a similar approach, but it cannot parallelize all query types. Ideally multi-tenant query federation should support the full PromQL language and the algorithms necessary would differ per query functions and operators used. This approach was deemed as a fairly complex way to achieve that tenant query federation.\nChallenges Aggregate data without overlaps Challenge The series in different tenants might have exactly the same labels and hence potentially collide which each other.\nProposal In order to be able to always identify the tenant correctly, queries using multiple tenants should inject a tenant label named __tenant_id__ and its value containing the tenant ID into the results. A potentially existing label with the same name should be stored in a label with the prefix original_.\nLabel selectors containing the tenant label should behave like any other labels. This can be achieved by selecting the tenants used in a multi tenant query.\nExposure of feature to the user Challenge The tenant ID is currently read from the X-Scope-OrgID HTTP header. The tenant ID has those documented limitations of values being allowed.\nProposal For the query path a user should be able to specify a X-Scope-OrgID header with multiple tenant IDs. Multiple tenant IDs should then be propagating throughout out the system until it reaches the querier. The Queryable instance returned by the querier component, is wrapped by a mergeQueryable, which will aggregate the results from a Queryable per tenant and hence treated by the downstream components as a single tenant query.\nTo allow such queries to be processed we suggest that an experimental configuration flag -querier.tenant-federation.enabled will be added, which is switched off by default. Once enabled the value of the X-Scope-OrgID header should be interpreted as | separated list of tenant ids. Components which are not expecting multiple tenant ids (e.g. the ingress path) must signal an error if multiple are used.\nImplementing Limits, Fairness and Observability for Cross-Tenant queries Challenge In Cortex the tenant id is used as the primary identifier for those components:\nThe limits that apply to a certain query.\nThe query-frontend maintains a per tenant query queue to implement fairness.\nRelevant metrics about the query are exposed under a user label.\nHaving a query spanning multiple tenants, the existing methods are no longer correct.\nProposal The identifier for aforementioned features for queries involving more than a single tenant should be derived from: An ordered, distinct list of tenant IDs, which is joined by a |. This will produce a reproducible identifier for the same set of tenants no matter which order they have been specified.\nWhile this feature is considered experimental, this provides some insights and ability to limit multi-tenant queries with these short comings:\nCardinality costs to the possible amount of tenant ID combinations.\nQuery limits applied to single tenants part of a multi tenant query are ignored.\nConclusion Challenge Status Aggregate data without overlap Implementation in PR #3250 Exposure of feature to the user Implementation in PR #3250 Implementing Limits, Fairness and Observability for Cross-Tenant queries Implementation in PR #3250 Future work Those features are currently out of scope for this proposal, but we can foresee some interest implementing them after this proposal.\nCross Tenant support for the ruler Ability to use multi tenant queries in the ruler.\nAllow the identifier for limits, fairness and observability to be switched out It would be great if the source identifier could be made more pluggable. This could allow to for example base all of those features on another dimension (e.g. users rather than tenants)\nAllow customisation of the label used to expose tenant ids As per this proposal the label name __tenant_id__ is fixed, but users might want to be able to modify that through a configuration option.\nRetain overlapping tenant id label values recursively As per this proposal the tenant label injection retains an existing label value, but this is not implemented recursively. So if the result already contains __tenant_id__ and original__tenant_id__ labels, the value of the latter would be lost.\n","categories":"","description":"","excerpt":" Author: Christian Simon Date: October 2020 Status: Accepted Overview …","ref":"/docs/proposals/cross-tenant-query-federation/","tags":"","title":"Cross-Tenant Query Federation"},{"body":" Author: Peter Stibrany Date: November 2020 Status: Accepted Deletion of tenant data Problem When a tenant is deleted from the external system that controls access to Cortex, we want to clean up tenants data from Cortex as well.\nBackground When using blocks storage, Cortex stores tenant’s data in several places:\nobject store for long-term storage of blocks, ingesters disks for short-term storage. Ingesters eventually upload data to long-term storage, various caches: query-frontend, chunks, index and metadata, object store for rules (separate from blocks), object store for alert manager configuration, state for alertmanager instances (notifications and silences). This document expects that there is an external authorization system in place. Disabling or deleting the tenant in this system will stop tenants data, queries and other API calls from reaching Cortex. Note that there may be a delay before disabling or deleting the user, until data / queries fully stop, due to eg. caching in authorization proxies. Cortex endpoint for deleting a tenant data should be only called after this blocking is in place.\nProposal API Endpoints POST /purger/delete_tenant We propose to introduce an /purger/delete_tenant API endpoint to trigger data deletion for the tenant.\nWhile this endpoint works as an “admin” endpoint and should not be exposed directly to tenants, it still needs to know under which tenant it should operate. For consistency with other endpoints this endpoint would therefore require an X-Scope-OrgID header and use the tenant from it.\nIt is safe to call “delete_tenant” multiple times.\nGET /purger/delete_tenant_status To monitor the state of data deletion, another endpoint will be available: /purger/delete_tenant_status. This will return OK if tenant’s data has been fully deleted – no more blocks exist on the long-term storage, alertmanager and rulers are fully stopped and configuration removed. Similarly to “delete_tenant” endpoint, “delete_tenant_status” will require X-Scope-OrgID header.\nImplementation (asynchronous) Purger will implement both API endpoints. Upon receiving the call to /purger/delete_tenant endpoint, Purger will initiate the deletion by writing “deletion marker” objects for specific tenant to following buckets:\nBlocks bucket Ruler configuration bucket Alertmanager configuration bucket Deletion marker for the tenant will be an object stored under tenant prefix, eg. “//deleted”. This object will contain the timestamp when it was created, so that we can delete it later based on the specified time period. We could reuse a subset of the proposed Thanos tombstones format, or use custom format.\n“delete_tenant_status” endpoint will report success, if all of the following are true:\nThere are no blocks for the tenant in the blocks bucket There is a “deletion finished” object in the bucket for the tenant in the ruler configuration bucket. There is a “deletion finished” object in the bucket for the tenant in the alertmanager configuration bucket. See later sections on Ruler and Alertmanager for explanation of “deletion finished” objects.\nBlocks deletion marker Blocks deletion marker will be used by compactor, querier and store-gateway.\nCompactor Upon discovering the blocks deletion marker, the compactor will start deletion of all blocks that belong to the tenant. This can take hours to finish, depending on the number of blocks. Even after deleting all blocks on the storage, ingesters may upload additional blocks for the tenant. To make sure that the compactor deletes these blocks, the compactor will keep the deletion marker in the bucket. After a configurable period of time it can delete the deletion marker too.\nTo implement deletion, Compactor should use new TenantsCleaner component similar to existing BlocksCleaner (which deletes blocks marked for deletion), and modify UserScanner to ignore deleted tenants (for BlocksCleaner and Compactor itself) or only return deleted tenants (for TenantsCleaner).\nQuerier, Store-gateway These two components scan for all tenants periodically, and can use the tenant deletion marker to skip tenants and avoid loading their blocks into memory and caching to disk (store-gateways). By implementing this, store-gateways will also unload and remove cached blocks.\nQueriers and store-gateways use metadata cache and chunks cache when accessing blocks to reduce the number of API calls to object storage. In this proposal we don’t suggest removing obsolete entries from the cache – we will instead rely on configured expiration time for cache items.\nNote: assuming no queries will be sent to the system once the tenant is deleted, implementing support for tenant deletion marker in queriers and store-gateways is just an optimization. These components will unload blocks once they are deleted from the object store even without this optimization.\nQuery Frontend While query-frontend could use the tenant blocks deletion marker to clean up the cache, we don’t suggest to do that due to additional complexity. Instead we will only rely on eventual eviction of cached query results from the cache. It is possible to configure Cortex to set TTL for items in the frontend cache by using -frontend.default-validity option.\nIngester Ingesters don’t scan object store for tenants.\nTo clean up the local state on ingesters, we will implement closing and deletion of local per-tenant data for idle TSDB. (See Cortex PR #3491). This requires additional configuration for ingesters, specifically how long to wait before closing and deleting TSDB. This feature needs to work properly in two different scenarios:\nIngester is no longer receiving data due to ring changes (eg. scale up of ingesters) Data is received because user has been deleted. Ingester doesn’t distinguish between the two at the moment. To make sure we don’t break queries by accidentally deleting TSDB too early, ingester needs to wait at least -querier.query-ingesters-within duration.\nAlternatively, ingester could check whether deletion marker exists on the block storage, when it detects idle TSDB.\nIf more data is pushed to the ingester for a given tenant, ingester will open new TSDB, build new blocks and upload them. It is therefore essential that no more data is pushed to Cortex for the tenant after calling the “delete_tenant” endpoint.\nRuler deletion marker This deletion marker is stored in the ruler configuration bucket. When rulers discover this marker during the periodic sync of the rule groups, they will\nstop the evaluation of the rule groups for the user, delete tenant rule groups (when multiple rulers do this at the same time, they will race for deletion, and need to be prepared to handle possible “object not found” errors) delete local state. When the ruler is finished with this cleanup, it will ask all other rulers if they still have any data for the tenant. If all other rulers reply with “no”, ruler can write “deletion finished” marker back to the bucket. This allows rulers to ignore the ruler completely, and it also communicates the status of the deletion back to purger.\nNote that ruler currently relies on cached local files when using Prometheus Ruler Manager component. This can be avoided now, and since it makes cleanup simpler, we suggest to modify Cortex ruler implementation to avoid this local copy.\nSimilarly to ingesters, it is necessary to disable access to the ruler API for deleted tenant. This must be done in external authorization proxy.\nAlertmanager deletion marker Deletion marker for alert manager is stored in the alertmanager configuration bucket. Cleanup procedure for alertmanager data is similar to rulers – when individual alertmanager instances discover the marker, they will:\nDelete tenant configuration Delete local notifications and silences state Ask other alertmanagers if they have any tenant state yet, and if not, write “deletion finished” marker back to the bucket. To perform the last step, Alertmanagers need to find other alertmanager instances. This will be implemented by using the ring, which will (likely) be added as per Alertmanager scaling proposal.\nAccess to Alertmanager API must be disabled for tenant that is going to be deleted.\nAlternatives Considered Another possibility how to deal with tenant data deletion is to make purger component actively communicate with ingester, compactor, ruler and alertmanagers to make data deletion faster. In this case purger would need to understand how to reach out to all those components (with multiple ring configurations, one for each component type), and internal API calls would need to have strict semantics around when the data deletion is complete. This alternative has been rejected due to additional complexity and only small benefit in terms of how fast data would be deleted.\n","categories":"","description":"","excerpt":" Author: Peter Stibrany Date: November 2020 Status: Accepted Deletion …","ref":"/docs/proposals/tenant-deletion/","tags":"","title":"Deletion of Tenant Data from Blocks Storage"},{"body":" Cortex Cortex is a horizontally scalable, highly available, multi-tenant, long term storage for Prometheus.\nFeatures Horizontally scalable: Cortex can run across multiple machines in a cluster, exceeding the throughput and storage of a single machine. Highly available: When run in a cluster, Cortex can replicate data between machines. Multi-tenant: Cortex can isolate data and queries from multiple different independent Prometheus sources in a single cluster. Long term storage: Cortex supports S3, GCS, Swift and Microsoft Azure for long term storage of metric data. Getting Started To get started with Cortex, follow these steps:\nInstallation Basic Setup Example Usage Documentation Architecture Overview Configuration Guide Security Guide Contributor Guidelines Community and Support If you have any questions about Cortex, you can:\nAsk a question on the Cortex Slack channel. To invite yourself to the CNCF Slack, visit http://slack.cncf.io/. File an issue. Email cortex-users@lists.cncf.io. Your feedback is always welcome.\nFor security issues see https://github.com/cortexproject/cortex/security/policy\nEngage with Our Community We invite you to participate in the Cortex Community Calls, an exciting opportunity to connect with fellow developers and enthusiasts. These meetings are held every 4 weeks on Thursdays at 1700 UTC, providing a platform for open discussion, collaboration, and knowledge sharing.\nOur meeting notes are meticulously documented and can be accessed here, offering a comprehensive overview of the topics discussed and decisions made.\nTo ensure you never miss a meeting, we’ve made it easy for you to keep track:\nView the Cortex Community Call schedule in your browser here. Alternatively, download the .ics file here for use with any calendar application or service that supports the iCal format. Join us in shaping the future of Cortex, and let’s build something amazing together!\nFurther reading Talks Nov 2024 KubeCon talk “Cortex Intro: Multi-Tenant Scalable Prometheus” (video, slides) Mar 2024 KubeCon talk “Cortex Intro: Multi-Tenant Scalable Prometheus” (video, slides) Apr 2023 KubeCon talk “How to Run a Rock Solid Multi-Tenant Prometheus” (video, slides) Oct 2022 KubeCon talk “Current State and the Future of Cortex” (video, slides) Oct 2021 KubeCon talk “Cortex: Intro and Production Tips” (video, slides) Sep 2020 KubeCon talk “Scaling Prometheus: How We Got Some Thanos Into Cortex” (video, slides) Jul 2020 PromCon talk “Sharing is Caring: Leveraging Open Source to Improve Cortex \u0026 Thanos” (video, slides) Nov 2019 KubeCon talks “Cortex 101: Horizontally Scalable Long Term Storage for Prometheus” (video, slides), “Configuring Cortex for Max Performance” (video, slides, write up) and “Blazin’ Fast PromQL” (slides, video, write up) Nov 2019 PromCon talk “Two Households, Both Alike in Dignity: Cortex and Thanos” (video, slides, write up) May 2019 KubeCon talks; “Cortex: Intro” (video, slides, blog post) and “Cortex: Deep Dive” (video, slides) Nov 2018 CloudNative London meetup talk; “Cortex: Horizontally Scalable, Highly Available Prometheus” (slides) Aug 2018 PromCon panel; “Prometheus Long-Term Storage Approaches” (video) Dec 2018 KubeCon talk; “Cortex: Infinitely Scalable Prometheus” (video, slides) Aug 2017 PromCon talk; “Cortex: Prometheus as a Service, One Year On” (videos, slides, write up part 1, part 2, part 3) Jun 2017 Prometheus London meetup talk; “Cortex: open-source, horizontally-scalable, distributed Prometheus” (video) Dec 2016 KubeCon talk; “Weave Cortex: Multi-tenant, horizontally scalable Prometheus as a Service” (video, slides) Aug 2016 PromCon talk; “Project Frankenstein: Multitenant, Scale-Out Prometheus”: (video, slides) Blog Posts Dec 2020 blog post “How AWS and Grafana Labs are scaling Cortex for the cloud” Oct 2020 blog post “How to switch Cortex from chunks to blocks storage (and why you won’t look back)” Oct 2020 blog post “Now GA: Cortex blocks storage for running Prometheus at scale with reduced operational complexity” Sep 2020 blog post “A Tale of Tail Latencies” Aug 2020 blog post “Scaling Prometheus: How we’re pushing Cortex blocks storage to its limit and beyond” Jul 2020 blog post “How blocks storage in Cortex reduces operational complexity for running Prometheus at massive scale” Mar 2020 blog post “Cortex: Zone Aware Replication” Mar 2020 blog post “How we’re using gossip to improve Cortex and Loki availability” Jan 2020 blog post “The Future of Cortex: Into the Next Decade” Feb 2019 blog post \u0026 podcast; “Prometheus Scalability with Bryan Boreham” (podcast) Feb 2019 blog post; “How Aspen Mesh Runs Cortex in Production” Dec 2018 CNCF blog post; “Cortex: a multi-tenant, horizontally scalable Prometheus-as-a-Service” Nov 2018 CNCF TOC Presentation; “Horizontally Scalable, Multi-tenant Prometheus” (slides) Sept 2018 blog post; “What is Cortex?” Jul 2018 design doc; “Cortex Query Optimisations” Jun 2016 design document; “Project Frankenstein: A Multi Tenant, Scale Out Prometheus” Hosted Cortex There are several commercial services where you can use Cortex on-demand:\nAmazon Managed Service for Prometheus (AMP) Amazon Managed Service for Prometheus (AMP) is a Prometheus-compatible monitoring service that makes it easy to monitor containerized applications at scale. It is a highly available, secure, and managed monitoring for your containers. Get started here. To learn more about the AMP, reference our documentation and Getting Started with AMP blog.\nEmeritus Maintainers Peter Štibraný @pstibrany Marco Pracucci @pracucci Bryan Boreham @bboreham Goutham Veeramachaneni @gouthamve Jacob Lisi @jtlisi Tom Wilkie @tomwilkie Alvin Lin @alvinlin123 History of Cortex The Cortex project was started by Tom Wilkie and Julius Volz (Prometheus’ co-founder) in June 2016.\n","categories":"","description":"","excerpt":" Cortex Cortex is a horizontally scalable, highly available, …","ref":"/docs/","tags":"","title":"Documentation"},{"body":" Author: Jay Batra Date: March 2020 Status: proposal Problem In Cortex, currently, we are missing versioning of documentation. The idea is to have version documentation just like Prometheus.Prometheus. Documentation is the main source of information for current contributors and first-timers. A properly versioned documentation will help everyone to have a proper place to look for answers before flagging it in the community.\nIn this proposal, we want to solve this. In particular, we want to:\nVersion specific pages of the documentation Include links to change version (the version must be in the URL) Include the master version and last 3 minor releases. Documentation defaults to the last minor release. Proposed solution Currently, the documentation is residing under the docs/ folder of cortexproject/cortex. It is built by Hugo using the theme docsy. It will have a proper drop-down menu which will enable proper versioning. It has a section params.version in config.toml which will allow us to map URLs with proper versions. We will have to change all the occurrences of older doc links with new links. We will keep master version with 3 latest release versions. Each release is a minor version expressed as 1.x. The document would default to latest minor version.\nFrom the current doc, the following paths (and all their subpages) should be versioned for now:\nhttps://cortexmetrics.io/docs/apis/ https://cortexmetrics.io/docs/configuration/ (moving v1.x Guarantees outside of the tree, because these shouldn’t be versioned) The above should be versioned under a single URL path (/docs/running-cortex/ in the following example, but final prefix is still to be decided).\nExample: For master version we would be able to use the above links via the following path\n/docs/running-cortex/master/configuration/ /docs/running-cortex/master/api/ And for a minor version like 1.x:\n/docs/running-cortex/1.0/configuration/ /docs/running-cortex/1.0/apis/ we’ll have versioned documentation only under the /docs/running-cortex/ prefix and, as a starting point, all versioned pages should go there.\n","categories":"","description":"","excerpt":" Author: Jay Batra Date: March 2020 Status: proposal Problem In …","ref":"/docs/proposals/documentation-versioning/","tags":"","title":"Documentation Versioning"},{"body":" Author: @annanay25 Reviewers: @jtlisi, @pstibrany, @cyriltovena, @pracucci Date: April 2020 Status: Accepted Overview Cortex uses modules to start and operate services with dependencies. Inter-service dependencies are specified in a map and passed to a module manager which ensures that they are initialised in the right order of dependencies. While this works really well, the implementation is tied in specifically to the Cortex struct and is not flexible for use with other projects like Loki, which also require similar forms of dependency management.\nWe would like to extend modules in cortex to a generic dependency management framework, that can be used by any project with no ties to cortex.\nSpecific goals Framework should allow for reusing cortex modules and allow us to: Add new modules Overwrite the implementation of a current module Manage dependencies Framework should allow for building an application from scratch using the modules package, with no dependencies on Cortex. For ex: Remove code from Loki that was copied from pkg/cortex/cortex.go. Proposed Design Modules package To make the modules package extensible, we need to abstract away any Cortex specific details from the module manager. The proposed design is to:\nMake a new component Manager, which is envisioned to be a central manager for all modules of the application. It stores modules \u0026 dependencies, and will be housed under a new package pkg/util/modules. Manager has the following methods for interaction: func (m *Manager) RegisterModule(name string, initFn func() (Service, error)) func (m *Manager) AddDependency(name string, dependsOn... string) error func (m *Manager) InitModuleServices(target string) (map[string]services.Service, error) Modules can be created by the application and registered with modules.Manager using RegisterModule. The parameters are:\nname: Name of the module initFn: A function that will be used to start the module. If it returns nil, and other modules depend on it, InitModuleServices will return an error. Dependencies between modules can be added using AddDependency. The parameters to the function are:\nname: Name of the module dependsOn: A variadic list of modules that the module depends on. These need to be added before the call to InitModuleServices.\nThe application can be initialized by running initFn’s of all the modules in the right order of dependencies by invoking InitModuleServices with the target module name.\nChanges to pkg/cortex: WrappedService present in the current module design will be deprecated. All initFn’s will be wrapped into WrappedService by default.\nWhile the process of loading modules into modules.Manager should be remain as part of the Cortex.New() function, InitModuleServices should be part of Cortex.Run() and to enable this, modules.Manager would be made a member of the Cortex struct.\nUsage Following these changes, the Modules package will be a generic dependency management framework that can be used by any project.\nTo use the modules framework: Import the pkg/util/modules package, and initialize a new instance of the Manager using modules.NewManager() Create components in the system that implement the services interface (present in pkg/util/services). Register each of these components as a module using Manager.RegisterModule() by passing name of the module and initFn for the module. To add dependencies between modules, use Manager.AddDependency() Once all modules are added into modules.Manager, initialize the application by calling Manager.InitModuleServices() which initializes modules in the right order of dependencies. Future work Extend the module manager to allow specifying multiple targets as opposed to a single target name supported currently. Factor out Run() method to make it independent of Cortex. This will help reduce replicated code in the Loki project as well as help manage modules.Manager outside of the Cortex struct. ","categories":"","description":"","excerpt":" Author: @annanay25 Reviewers: @jtlisi, @pstibrany, @cyriltovena, …","ref":"/docs/proposals/generalize-modules/","tags":"","title":"Generalize Modules Service to make it extensible"},{"body":"Cortex is a powerful platform software that can be run in two modes: as a single binary or as multiple independent microservices.\nThere are two guides in this section:\nSingle Binary Mode with Docker Compose Microservice Mode with KIND The single binary mode is useful for testing and development, while the microservice mode is useful for production.\nBoth guides will help you get started with Cortex using blocks storage.\nSingle Binary Mode This guide will help you get started with Cortex in single-binary mode using blocks storage.\nPrerequisites Cortex can be configured to use local storage or cloud storage (S3, GCS, and Azure). It can also utilize external Memcached and Redis instances for caching. This guide will focus on running Cortex as a single process with no dependencies.\nDocker Compose Running Cortex as a Single Instance For simplicity, we’ll start by running Cortex as a single process with no dependencies. This mode is not recommended or intended for production environments or production use.\nThis example uses Docker Compose to set up:\nAn instance of SeaweedFS for S3-compatible object storage An instance of Cortex to receive metrics. An instance of Prometheus to send metrics to Cortex. An instance of Grafana to visualize the metrics. Instructions $ git clone https://github.com/cortexproject/cortex.git $ cd cortex/docs/getting-started Start the services # Start SeaweedFS to emulate S3 storage for Cortex. $ docker-compose up seaweedfs -d --wait # Create buckets in SeaweedFS. $ for bucket in cortex-blocks cortex-ruler cortex-alertmanager; do curl --aws-sigv4 \"aws:amz:local:seaweedfs\" --user \"any:any\" -X PUT http://localhost:8333/$bucket done # Start the remaining services. $ docker-compose up -d --wait We can now access the following services:\nCortex Prometheus Grafana SeaweedFS If everything is working correctly, Prometheus should be sending metrics that it is scraping to Cortex. Prometheus is configured to send metrics to Cortex via remote_write. Check out the prometheus-config.yaml file to see how this is configured.\nConfigure Cortex Recording Rules and Alerting Rules We can configure Cortex with cortextool to load recording rules and alerting rules. This is optional, but it is helpful to see how Cortex can be configured to manage rules and alerts.\n# Configure recording rules for the Cortex tenant (optional) $ docker run --network host -v \"$(pwd):/workspace\" -w /workspace quay.io/cortexproject/cortex-tools:v0.17.0 rules sync rules.yaml alerts.yaml --id cortex --address http://localhost:9009 Configure Cortex Alertmanager Cortex also comes with a multi-tenant Alertmanager. Let’s load configuration for it to be able to view them in Grafana.\n# Configure alertmanager for the Cortex tenant $ docker run --network host -v \"$(pwd):/workspace\" -w /workspace quay.io/cortexproject/cortex-tools:v0.17.0 alertmanager load alertmanager-config.yaml --id cortex --address http://localhost:9009 You can configure Alertmanager in Grafana as well.\nThere’s a list of recording rules and alerts that should be visible in Grafana here.\nExplore Grafana is configured to use Cortex as a data source. Grafana is also configured with Cortex Dashboards to understand the state of the Cortex instance. The dashboards are generated from the cortex-jsonnet repository. There is a Makefile in the repository that can be used to update the dashboards.\n# Update the dashboards (optional) $ make If everything is working correctly, then the metrics seen in Grafana were successfully sent from Prometheus to Cortex via remote_write!\nOther things to explore:\nCortex - Administrative interface for Cortex Try shutting down the ingester, and see how it affects metric ingestion. Restart Cortex to bring the ingester back online, and see how Prometheus catches up. Does it affect the querying of metrics in Grafana? Prometheus - Prometheus instance that is sending metrics to Cortex Try querying the metrics in Prometheus. Are they the same as what you see in Cortex? Grafana - Grafana instance that is visualizing the metrics. Try creating a new dashboard and adding a new panel with a query to Cortex. Clean up $ docker-compose down Microservice Mode Now that you have Cortex running as a single instance, let’s explore how to run Cortex in microservice mode.\nPrerequisites Kind kubectl Helm This example uses Kind to set up:\nA Kubernetes cluster An instance of SeaweedFS for S3-compatible object storage An instance of Cortex to receive metrics An instance of Prometheus to send metrics to Cortex An instance of Grafana to visualize the metrics Setup Kind $ kind create cluster Configure Helm $ helm repo add cortex-helm https://cortexproject.github.io/cortex-helm-chart $ helm repo add grafana https://grafana.github.io/helm-charts $ helm repo add prometheus-community https://prometheus-community.github.io/helm-charts Instructions $ git clone https://github.com/cortexproject/cortex.git $ cd cortex/docs/getting-started Configure SeaweedFS (S3) # Create a namespace $ kubectl create namespace cortex # We can emulate S3 with SeaweedFS $ kubectl --namespace cortex apply -f seaweedfs.yaml --wait --timeout=5m # Wait for SeaweedFS to be ready $ sleep 5 $ kubectl --namespace cortex wait --for=condition=ready pod -l app=seaweedfs --timeout=5m # Port-forward to SeaweedFS to create a bucket $ kubectl --namespace cortex port-forward svc/seaweedfs 8333 \u0026 # Create buckets in SeaweedFS $ for bucket in cortex-blocks cortex-ruler cortex-alertmanager; do curl --aws-sigv4 \"aws:amz:local:seaweedfs\" --user \"any:any\" -X PUT http://localhost:8333/$bucket done Setup Cortex # Deploy Cortex using the provided values file which configures # - blocks storage to use the seaweedfs service $ helm upgrade --install --version=2.4.0 --namespace cortex cortex cortex-helm/cortex -f cortex-values.yaml --wait Setup Prometheus # Deploy Prometheus to scrape metrics in the cluster and send them, via remote_write, to Cortex. $ helm upgrade --install --version=25.20.1 --namespace cortex prometheus prometheus-community/prometheus -f prometheus-values.yaml --wait If everything is working correctly, Prometheus should be sending metrics that it is scraping to Cortex. Prometheus is configured to send metrics to Cortex via remote_write. Check out the prometheus-config.yaml file to see how this is configured.\nSetup Grafana # Deploy Grafana to visualize the metrics that were sent to Cortex. $ helm upgrade --install --version=7.3.9 --namespace cortex grafana grafana/grafana -f grafana-values.yaml --wait # Create dashboards for Cortex $ for dashboard in $(ls dashboards); do basename=$(basename -s .json $dashboard) cmname=grafana-dashboard-$basename kubectl create --namespace cortex configmap $cmname --from-file=$dashboard=dashboards/$dashboard --save-config=true -o yaml --dry-run=client | kubectl apply -f - kubectl patch --namespace cortex configmap $cmname -p '{\"metadata\":{\"labels\":{\"grafana_dashboard\":\"\"}}}' done # Port-forward to Grafana to visualize $ kubectl --namespace cortex port-forward deploy/grafana 3000 \u0026 View the dashboards in Grafana.\nConfigure Cortex Recording Rules and Alerting Rules (Optional) We can configure Cortex with cortextool to load recording rules and alerting rules. This is optional, but it is helpful to see how Cortex can be configured to manage rules and alerts.\n# Port forward to the alertmanager to configure recording rules and alerts $ kubectl --namespace cortex port-forward svc/cortex-nginx 8080:80 \u0026 # Configure recording rules for the cortex tenant $ cortextool rules sync rules.yaml alerts.yaml --id cortex --address http://localhost:8080 Configure Cortex Alertmanager (Optional) Cortex also comes with a multi-tenant Alertmanager. Let’s load configuration for it to be able to view them in Grafana.\n# Configure alertmanager for the cortex tenant $ cortextool alertmanager load alertmanager-config.yaml --id cortex --address http://localhost:8080 You can configure Alertmanager in Grafana as well.\nThere’s a list of recording rules and alerts that should be visible in Grafana here.\nExplore Grafana is configured to use Cortex as a data source. Grafana is also configured with Cortex Dashboards to understand the state of the Cortex instance. The dashboards are generated from the cortex-jsonnet repository. There is a Makefile in the repository that can be used to update the dashboards.\n# Update the dashboards (optional) $ make If everything is working correctly, then the metrics seen in Grafana were successfully sent from Prometheus to Cortex via remote_write!\nOther things to explore:\nCortex - Administrative interface for Cortex\n# Port forward to the ingester to see the administrative interface for Cortex $ kubectl --namespace cortex port-forward deploy/cortex-ingester 9009:8080 \u0026 Try shutting down the ingester, and see how it affects metric ingestion. Restart Cortex to bring the ingester back online, and see how Prometheus catches up. Does it affect the querying of metrics in Grafana? Prometheus - Prometheus instance that is sending metrics to Cortex\n# Port forward to Prometheus to see the metrics that are being scraped $ kubectl --namespace cortex port-forward deploy/prometheus-server 9090 \u0026 Try querying the metrics in Prometheus. Are they the same as what you see in Cortex? Grafana - Grafana instance that is visualizing the metrics.\n# Port forward to Grafana to visualize $ kubectl --namespace cortex port-forward deploy/grafana 3000 \u0026 Try creating a new dashboard and adding a new panel with a query to Cortex. Clean up # Remove the port-forwards $ killall kubectl $ kind delete cluster ","categories":"","description":"","excerpt":"Cortex is a powerful platform software that can be run in two modes: …","ref":"/docs/getting-started/","tags":"","title":"Getting Started"},{"body":"Gojek launched in 2010 as a call center for booking motorcycle taxi rides in Indonesia. Today, the startup is a decacorn serving millions of users across Southeast Asia with its mobile wallet, GoPay, and 20+ products on its super app. Want to order dinner? Book a massage? Buy movie tickets? You can do all of that with the Gojek app.\nThe company’s mission is to solve everyday challenges with technology innovation. To achieve that across multiple markets the systems team at Gojek focused on building an infrastructure for speed, reliability, and scale. By 2019, the team realized it needed a new monitoring system that could keep up with Gojek’s ever-growing technology organization, which led them to Cortex, the horizontally scalable Prometheus implementation.\n“We were using InfluxDB for metrics storage. Developers configured alerts by committing kapacitor scripts in git repos. To achieve high availability, we had a relay setup with two InfluxDBs. Since we could not horizontally scale Influx unless we paid for an enterprise license, we ended up having many InfluxDB clusters with relay setup,” says Product Engineer Ankit Goel.\nThough the team had introduced automation for setup, managing all those Influx instances became a pain point for operations. Additionally, some of the Gojek engineering teams needed far greater scale. “Some of our teams generate more than a million active time series,” says Goel. Another common requirement from customers was long-term storage of metrics. With InfluxDB, Gojek only had 2 weeks’ retention, and increasing it would mean provisioning bigger instances.\nGojek was in search of a better monitoring solution that would meet the following requirements:\nKubernetes native. Horizontally scalable. Highly available out of the box. High reliability. Low operations overhead so a small team can manage it. Cortex met all of these requirements, and also had the following features that the Gojek team could leverage:\nMulti-tenancy. Customizable and modifiable, so it could be integrated with Gojek’s existing tooling. Support for remote_write. Because it supports remote_write, Cortex enabled one of Gojek’s key needs: the ability to offer monitoring as a service. “With Thanos, we would have had to deploy a Thanos sidecar on every Prometheus that would have been deployed,” says Goel. “So essentially, there would be a substantial part of infrastructure on the client side that we would need to manage. We preferred Cortex because people could simply push their metrics to us, and we would have all the metrics in a single place.”\nThe implementation started in January 2019. The team developed a few tools: a simple service for token-based authentication, and another for storing team information, such as notification channels and PagerDuty policies. Once all this was done, they leveraged InfluxData Telegraf’s remote_write plugin to write to Cortex. This allowed them to have all the metrics being sent to InfluxDB to be sent to Cortex as well. “So moving from InfluxDB to tenants would not be that complicated. Because Cortex was multi-tenant, we could directly map each of our InfluxDB servers to our tenants,” says Goel. They’ve developed an internal helm chart to deploy and manage Cortex. After the customizations were completed in about two months, “we had our setup up and running, and we onboarded one team on Cortex,” he says.\nIn the initial version, GitOps was the only interface for developers to apply alerts and create dashboards. The team built tools like grafonnet-playground to make it easy for developers to create dashboards. Developers are also allowed to create dashboards using the UI, since Grafana maintains version history for dashboards.\n“We needed metrics like ‘the number of alerts triggered for each team,’ ‘how long did it take to resolve these alerts,’ ‘how many were actionable and how many were ignored,’ etc.,” says Goel. “For measuring these metrics, the team only had to create a simple dashboard, since the ruler component exposes the per-tenant alert metrics. Both business and developers have found these metrics to be very useful.”\nThe team built a CLI tool to improve user experience for applying alerts without having to dig into PromQL. “You can write a command and say lens attach alert, and you tell it what kind of alert you want to attach, such as a CPU alert or Postgres alerts, and then you give it a service name,” says Goel. “There are some challenges to this approach for applying alerts, but we would like to move to such a model in the future.”\nOne of the challenges the team faces is developer education. But “we always knew if we are going to move to either Thanos or Cortex, developers would have to learn PromQL,” Goel says. The monitoring team paired with developers to help them understand PromQL and migrate their graphs and alerts.\nThe monitoring team has faced issues with Cortex from time to time, but “we always reached out to the Cortex community with our issues through the Cortex slack channel,” says Goel, and “active members of the Cortex community have always helped us with our problems.”\nToday, Gojek’s Lens monitoring system has 40+ tenants, for which Cortex handles about 1.2 million samples per second. Adoption is growing organically by word of mouth. Gojek is currently migrating to Kubernetes, and the teams that moved to Kubernetes have found Prometheus to be a better fit than InfluxDB. Seeing that success, other teams on Kubernetes have onboarded themselves to Lens.\nUltimately, Goel says, “where Cortex has really helped us is to integrate the monitoring system with our existing tools. We have a lot of internal tooling, and in certain places, we needed really tight integrations with the monitoring system. So the goal is to make sure that whenever a new service or team is created, they automatically get onboarded to the monitoring platform. After developers deploy, some of their system metrics and all the other standard metrics that are available for a service are automatically sent to the platform.” The team plans to spend the next six months bringing everyone over to Lens.\nLooking ahead, Goel and his team have the long-term vision of growing from a monitoring team to a full-fledged observability team. “We also want to take care of logging and tracing in Gojek,” he says. “Loki would be easy to fit with Cortex, so in the future we want to explore Loki for logging.”\n","categories":"","description":"","excerpt":"Gojek launched in 2010 as a call center for booking motorcycle taxi …","ref":"/docs/case-studies/gojek/","tags":"","title":"How Gojek Is Leveraging Cortex to Keep Up with Its Ever-Growing Scale"},{"body":"This document defines project governance for the project.\nVoting The Cortex project employs voting to ensure no single member can dominate the project. Any maintainer may cast a vote. To avoid having a single company dominate the project, at most two votes from maintainers working for the same company will count.\nFor formal votes, a specific statement of what is being voted on should be added to the relevant github issue or PR, and a link to that issue or PR added to the maintainers meeting agenda document.\nMaintainers should indicate their yes/no vote on that issue or PR, and after a suitable period of time (minimum 2 business weeks), the votes will be tallied and the outcome noted. Maintainers who do not cast a vote, after a suitable period of time, are not included in the majority calculation.\nMaintainer Role Maintainers are required to participate in the project, by joining discussions, submitting and reviewing pull requests, answering user questions, among others.\nBesides that, we have one concrete activity in which maintainers have to engage from time to time: releasing new versions of Cortex. This process ideally takes only a couple of hours, but requires coordination on different fronts. Even though the process is well documented, it is not without eventual glitches, so, each release needs a “Release shepherd”. How it works is described in the RELEASE.md file.\nChanges in Maintainership Contributors who are interested in becoming a maintainer, if performing relevant responsibilities, should discuss their interest with the existing maintainers. New maintainers must be nominated by an existing maintainer and must be elected by 2/3 majority vote.\nWe do not expect anyone to make a permanent commitment to be a Cortex maintainer forever. After all, circumstances change, people get new jobs, new interests, and may not be able to continue contributing to the project. At the same time, we need to keep the list of maintainers current in order to have effective governance. People may be removed from the current list of maintainers via one of the following ways:\nThey can resign If they stop contributing to the project for a period of 6 months or more By a 2/3 majority vote from active maintainers Former maintainers are recognized with an honorary Emeritus Maintainer status, and have their names permanently listed in the README as a form of gratitude for their contributions.\nApproving PRs PRs may be merged after receiving at least two positive votes. If the PR author is a maintainer, this counts as a vote.\nTriage Role Triagers are contributors who assist in managing and organizing issues and pull requests. See here for details about the role’s permission.\nGithub Project Administration Maintainers will be added to the collaborators list of the Cortex repository with “Write” access while Triagers will be added to the collaborators list of the Cortex repository with “Triage” access.\nAfter 6 months a maintainer will be given “Admin” access to the Cortex repository.\nChanges in Governance All changes in Governance require a 2/3 majority vote.\nCortex Team Maintainers Name Email GitHub Company Alan Protasio alanprot@gmail.com @alanprot Amazon Web Services Ben Ye yb532204897@gmail.com @yeya24 Amazon Web Services Charlie Le chtle4@gmail.com\u003e @CharlieTLe Apple Daniel Blando daniel@blando.com.br @danielblando Amazon Web Services Friedrich Gonzalez friedrichg@gmail.com @friedrichg Adobe Triaggers Name Email GitHub Company Sungjin Lee tjdwls1201@gmail.com @SungJin1212 KakaoEnterprise Other Changes Unless specified above, all other changes to the project require a 2/3 majority vote. Additionally, any maintainer may request that any change require a 2/3 majority vote.\n","categories":"","description":"","excerpt":"This document defines project governance for the project.\nVoting The …","ref":"/docs/contributing/governance/","tags":"","title":"Governance"},{"body":" Author: @jtlisi Reviewers: @pracucci, @pstibrany, @khaines, @gouthamve Date: March 2020 Status: Accepted Overview The purpose of this design document is to propose a set of standards that should be the basis of the Cortex HTTP API. This document will outline the current state of the Cortex http api and describe limitations that result from the current approach. It will also outline a set of paradigms on how http routes should be created within Cortex.\nCurrent Design As things currently stand, the majority of HTTP API calls exist under the /api/prom path prefix. This prefix is configurable. However, since this prefix is shared between all the modules which leads to conflicts if the Alertmanager is attempted to be run as as part of the single binary (#1722).\nProposed Design Module-Based Routing Cortex incorporates three separate APIs: Alertmanager, Prometheus, and Cortex. Each of these APIs should use a separate route prefix that accurately describes the API. Currently, all of the api calls in Cortex reside under the configured http prefix. Instead the following routing tree is proposed:\n/prometheus/* Under this path prefix, Cortex will act as a Prometheus web server. It will host all of the required Prometheus api endpoints. For example to query cortex the endpoint /prometheus/api/v1/query_range will be used.\n/alertmanager/* Under this path prefix, Cortex will act as a Alertmanager web server. In this case, it will forward requests to the alertmanager and support the alertmanager API. This means for a user to access their Alertmanager UI, they will use the /alertmanager path of cortex.\n/api/v1/* – The cortex API will exist under this path prefix. /push /chunks /rules/* Current Proposed /api/prom/push /api/v1/push /api/prom/chunks /api/v1/chunks /api/prom/rules/* /api/v1/rules/* Service Endpoints A number of endpoints currently exist that are not under the /api/prom prefix that provide basic web interfaces and trigger operations for cortex services. These endpoints will all be placed under a url with their service name as a prefix if it is applicable.\nCurrent Proposed /status /multitenant-alertmanager/status /config /config /ring /ingester/ring /ruler_ring /ruler/ring /compactor/ring /compactor/ring /store-gateway/ring /store-gateway/ring /ha-tracker /distributor/ha_tracker /all_user_stats /distributor/all_user_stats /user_stats /distributor/user_stats /flush /ingester/flush /shutdown /ingester/shutdown Path Versioning Cortex will utilize path based versioning similar to both Prometheus and Alertmanager. This will allow future versions of the API to be released with changes over time.\nBackwards-Compatibility The new API endpoints and the current http prefix endpoints can be maintained concurrently. The flag to configure these endpoints will be maintained as http.prefix. This will allow us to roll out the new API without disrupting the current routing schema. The original http prefix endpoints can maintained indefinitely or be phased out over time. Deprecation warnings can be added to the current API either when initialized or utilized. This can be accomplished by injecting a middleware that logs a warning whenever a legacy API endpoint is used.\nIn cases where Cortex is run as a single binary, the Alertmanager module will only be accessible using the new API.\nImplementation This will be implemented by adding an API module to the Cortex service. This module will handle setting up all the required HTTP routes with Cortex. It will be designed around a set of interfaces required to fulfill the API. This is similar to how the v1 Prometheus API is implemented.\nStyle All new paths will utilize _ instead of - for their url to conform with Prometheus and its use of the underscore in the query_range endpoint. This applies to all operations endpoints. Component names in the path can still contain dashes. For example: /store-gateway/ring. ","categories":"","description":"","excerpt":" Author: @jtlisi Reviewers: @pracucci, @pstibrany, @khaines, …","ref":"/docs/proposals/http-api-design/","tags":"","title":"HTTP API Design"},{"body":" Author: @pstibrany Reviewers: Date: June 2020 Status: Replaced with migration guide (now removed from this site). Warning Suggestions from this proposal were implemented, but general procedure outlined here doesn’t quite work in Kubernetes environment. Please see chunks to blocks migration guide (now removed from this site) instead.\nIntroduction This short document describes the first step in full migration of the Cortex cluster from using chunks storage to using blocks storage, specifically switching ingesters to using blocks, and modification of queriers to query both chunks and blocks storage.\nIngesters When switching ingesters from chunks to blocks, we need to consider the following:\nIngesting of new data, and querying should work during the switch. Ingesters are rolled out with new configuration over time. There is overlap: ingesters of both kinds (chunks, blocks) are running at the same time. Ingesters using WAL don’t flush in-memory chunks to storage on shutdown. Rollout should be as automated as possible. How do we handle ingesters with WAL (non-WAL ingesters are discussed below)? There are several possibilities, but the simplest option seems to be adding a new flag to ingesters to flush chunks on shutdown. This is trivial change to ingester, and allows us to do automated migration by:\nEnabling this flag on each ingester (first rollout). Turn off chunks, enable TSDB (second rollout). During the second rollout, as the ingester shuts down, it will flush all chunks in memory, and when it restarts, it will start using TSDB. Benefit of this approach is that it is trivial to add the flag, and then rollout in both steps can be fully automated. In this scenario, we will reconfigure existing statefulset of ingesters to use blocks in step 2.\nNotice that querier can ask only ingesters for most recent data and not consult the store, but during the rollout (and some time after), ingesters that are already using blocks will not have the most recent chunks in memory. To make sure queries work correctly, -querier.query-store-after needs to be set to 0, in order for queriers to not rely on ingesters only for most recent data. After couple of hours after rollout, this value can be increased again, depending on how much data ingesters keep. (-experimental.blocks-storage.tsdb.retention-period for blocks, -ingester.retain-period for chunks) During the rollout, chunks and blocks ingesters share the ring and use the same statefulset.\nOther alternatives considered for flushing chunks / handling WAL:\nReplay chunks-WAL into TSDB head on restart. In this scenario, chunks-ingester shuts down, and block ingester starts. It can detect existing chunks WAL, and replay it into TSDB head (and then delete old WAL). Issue here is that current chunks-WAL is quite specific to ingester code, and would require some refactoring to make this happen. Deployment is trivial: just reconfigure ingesters to start using blocks, and replay chunks WAL if found. Required change seems like a couple of days of coding work, but it is essentially only used once (for each cluster). Doesn’t seem like good time investment. Shutdown single chunks-ingester, run flusher in its place, and when done start new blocks ingester. This is similar to the procedure we did during the introduction of WAL. Flusher can be run via initContainer support in pods. This still requires two-step deployment: 1) enable flusher and reconfigure ingesters to use blocks, 2) remove flusher. When not using WAL, ingesters using chunks cannot transfer those chunks to new ingesters that start with blocks support, so old ingesters need to be configured to disable transfers (using -ingester.max-transfer-retries=0), and to flush chunks on shutdown instead. As ingesters without WAL are typically deployed using Kubernetes deployment, while blocks ingesters need to use statefulset, and there is no chunks transfer happening, it is possible to configure and start blocks-ingesters and then stop old deployment.\nAfter all ingesters are converted to blocks, we can set cut-off time for querying chunks storage on queriers.\nFor rollback from blocks to chunks, we need to be able to flush data from ingesters to the blocks storage, and then switch ingesters back to chunks. Ingesters are currently not able to flush blocks to storage, but adding flush-on-shutdown option, support for /shutdown endpoint and support in flusher component similar to chunks is doable, and should be part of this work.\nWith this ability, rollback would follow the same process, just in reverse: 1) redeploy with flush flag enabled, 2a) redeploy with config change from blocks to chunks (when using WAL) or 2b) scale down statefulset with blocks-ingesters, and start deployment with chunk-ingesters again. Note that this isn’t a full rollback to chunks-only solution, as generated blocks still need to be queried after the rollback, otherwise samples pushed to blocks would be missing. This means running store-gateways and queriers that can query both chunks and blocks store.\nAlternative plan could be to use a separate Cortex cluster configured to use blocks, and redirect incoming traffic to both chunks and blocks cluster. When one is confident about the blocks cluster running correctly, old chunks cluster can be shutdown. In this plan, there is an overlap where both clusters are ingesting same data. Blocks cluster needs to be configured to be able to query chunks storage as well, with cut-off time based on when clusters were configured (at latest, to minimize amount of duplicated samples that need to be processed during queries.)\nQuerying To be able to query both old and new data, querier needs to be modified to be able to query both blocks (on object store only) and chunks store (NoSQL + object store) at the same time, and merge results from both.\nFor querying chunks storage, we have two options:\nAlways query the chunks store – useful during ingesters switch, or after rollback from blocks to chunks. Query chunk store only for queries that ask for data after specific cut-off time. This is useful after all ingesters have switched, and we know the timestamp since ingesters are only writing blocks. Querier needs to support both modes of querying chunks store. Which one of these two modes is used depends on single timestamp flag passed to the querier. If timestamp is configured, chunks store is only used for queries that ask for data older than timestamp. If timestamp is not configured, chunks store is always queried.\nFor blocks, we don’t need to use the timestamp flag. Queriers can always query blocks – each querier knows about existing blocks and their timeranges, so it can quickly determine whether there are any blocks with relevant data. Always querying blocks is also useful when there is some background process converting chunks to blocks. As new blocks with old data appear on the store as a result of conversion, they get queried if necessary.\nWhile we could use runtime-config for on-the-fly switch without restarts, queriers restart quickly and so switching via configuration or command line option seems enough.\nWork to do Ingester: Add flags for always flushing on shutdown, even when using WAL or blocks. Querier: Add support for querying both chunk store and blocks at the same time and test the support for querying both chunks and blocks from ingesters works correctly Querier: Add cut-off time support to querier to query chunk the store only if needed, based on query time. ","categories":"","description":"","excerpt":" Author: @pstibrany Reviewers: Date: June 2020 Status: Replaced with …","ref":"/docs/proposals/ingesters-migration/","tags":"","title":"Migrating ingesters from chunks to blocks and back."},{"body":" Author: Roy Chiang Date: May 2021 Status: Proposed Introduction As a part of pushing Cortex’s scaling capability at AWS, we have done performance testing with Cortex and found the compactor to be one of the main limiting factors for higher active timeseries limit per tenant. The documentation Compactor describes the responsibilities of a compactor, and this proposal focuses on the limitations of the current compactor architecture. In the current architecture, compactor has simple sharding, meaning that a single tenant is sharded to a single compactor. The compactor generates compaction groups, which are groups of Prometheus TSDB blocks that can be compacted together, independently of another group. However, a compactor currently handles compaction groups of a single tenant iteratively, meaning that blocks belonging non-overlapping times are not compacted in parallel.\nCortex ingesters are responsible for uploading TSDB blocks with data emitted by a tenant. These blocks are considered as level-1 blocks, as they contain duplicate timeseries for the same time interval, depending on the replication factor. Vertical compaction is done to merge all the blocks with the same time interval and deduplicate the samples. These merged blocks are level-2 blocks. Subsequent compactions such as horizontal compaction can happen, further increasing the compaction level of the blocks.\nProblem and Requirements Currently, a compactor is able to compact up to 20M timeseries within 2 hours for a level-2 compaction, including the time to download blocks, compact, and upload the newly compacted block. We would like to increase the timeseries limit per tenant, and compaction is one of the limiting factors. In addition, we would like to achieve the following:\nCompact multiple non-overlapping time intervals concurrently, so we can achieve higher throughput for the compaction of a single tenant We should be able to scale up, down compactor as needed, depending on how many compactions are pending Insight into the compaction progress of a tenant, such as the number of compactions required in order to catch up to the newest blocks Design We accept the fact that a single compaction can potentially take more than 2 hours to compact, and we achieve higher compaction throughput through horizontally scaling the compactor. To compact more blocks in parallel for a single tenant, we distribute the compaction groups to compactors, instead of introducing more parallelism within a compactor.\nParallelize Work This proposal builds heavily on top of the GrafanaLabs approach of introducing parallelism via time intervals. The difference being that a single tenant is now sharded across multiple compactors instead of just a single compactor. The initial approach will be to work on distinct time intervals, but the compactor planner can be later extended to introduce parallelism within a time interval as well.\nThe following is an example of parallelize work at each level:\nCompactors are shuffle-sharded, meaning that 1 tenant can belong to multiple compactors, and these subset of compactors determine which blocks should be compacted together. Compactors determine amongst themselves the responsibility of the compaction blocks, by using a hash of time interval and tenant id, and putting it on the sharding ring.\nThe benefit of this approach is that this aligns with what Cortex currently does in Ruler. The downside is that a compaction job can only be assigned to a single compactor, rather than all of the compactors sharded for the tenant. If a compaction job takes forever, other tenants sharded to the same compactor will be blocked until the issue is resolved. With the scheduler approach, any compactor assigned to a given tenant can pick up any work required.\nScenarios Bad block resulting in non-ideal compaction groups A Cortex operator configures the compaction block range as 2h and 6h. If a full 6-hour block cannot be compacted due to compaction failures, the compactor should not split up the group into subgroups, as this may cause suboptimal grouping of block. Cortex has full information regarding all the available blocks, so we should utilize this information to achieve the best compaction group possible.\nAlternatives Shard compaction jobs amongst compactors with a scheduler We add a new component Compactor Scheduler, which is responsible for calculating the compaction plan, and distributing compaction groups to compactors. The planner is sharded by tenant id, so that we can horizontally scale the planner as needed in order to accept more tenants in the cortex cluster. A tenant will have two queues inside the planner, a compaction queue and a clean up queue, similar to how the query frontend currently holds queues of pending queries.\nOnce a compactor scheduler pushes a job to a compactor, the job is no longer available. Every set interval, or once the compaction is done, a compactor will update the compactor schedule the current status of the compaction job. If a compactor does not provide an update to the scheduler within a timeout, the compaction job becomes available to be assigned to other compactors.\nConcurrency To achieve concurrency within a single tenant, compactor scheduler will push jobs to compactors. Compactors are shuffle-sharded by tenant id, to prevent a large tenant from impacting the compaction of other tenants. Compactor will download blocks from long term storage, compact, and upload. Compactor will also pull from the clean up queues from scheduler, and delete blocks marked for deletion.\nConsistency On resharding of compactor schedulers, a tenant might move to a different scheduler. We can either drop the current compactor job in order to prevent duplicate compaction jobs, or continue compaction. I propose that the compactor drops the compaction job if the compaction group no longer belongs to the original compactor scheduler. This way, we do not have duplicate compactions happening, and we can minimize work wasted.\nContribute to Thanos for a more scalable compactor Instead of introducing parallelism on the Cortex compactor level, we move the parallelism to the Thanos compactor itself. Thanos has a proposal to make compactor more scalable, and a PR. Cortex will enjoy higher throughput per tenant if Thanos is able to speed up the compaction, and we can keep the Cortex architecture the same. However, this approach means that a single tenant is still sharded to a single compactor. In order to compact more groups at once, we must scale up compactor vertically. Although vertical scaling can get us far, we should scale horizontally where we can.\n","categories":"","description":"","excerpt":" Author: Roy Chiang Date: May 2021 Status: Proposed Introduction As a …","ref":"/docs/proposals/parallel-compaction/","tags":"","title":"Parallel Compaction by Time Interval"},{"body":" Author: Allenzhli Date: January 2021 Status: Accepted, Implemented in PR #3879. Retention of tenant data Problem Metric data is growing over time per-tenant, at the same time, the value of data decreases. We want to have a retention policy like prometheus does. In Cortex, data retention is typically achieved via a bucket policy. However, this has two main issues:\nNot every backend storage support bucket policies Bucket policies don’t easily allow a per-tenant custom retention Background tenants When using blocks storage, Cortex stores tenant’s data in object store for long-term storage of blocks, tenant id as part of the object store path. We discover all tenants via scan the root dir of bucket.\nruntime config Using the “overrides” mechanism (part of runtime config) already allows for per-tenant settings. See runtime-configuration-file for more details. Using it for tenant retention would fit nicely. Admin could set per-tenant retention here, and also have a single global value for tenants that don’t have custom value set.\nProposal retention period field We propose to introduce just one new field RetentionPeriod in the Limits struct(defined at pkg/util/validation/limits.go).\nRetentionPeriod setting how long historical metric data retention period per-tenant. 0 is disable.\nRuntime config is reloaded periodically (defaults to 10 seconds), so we can update the retention settings on-the-fly.\nFor each tenant, if a tenant-specific runtime_config value exists, it will be used directly, otherwise, if a default limits_config value exists, then the default value will be used; If neither exists, do nothing.\nImplementation A BlocksCleaner within the Compactor run periodically (which defaults to 15 minutes) and the retention logic will insert into it. The logic should compare retention value to block maxTime and blocks that match maxTime \u003c now - retention will be marked for delete.\nBlocks deletion is not immediate, but follows a two steps process. See soft-and-hard-blocks-deletion\n","categories":"","description":"","excerpt":" Author: Allenzhli Date: January 2021 Status: Accepted, Implemented in …","ref":"/docs/proposals/tenant-retention/","tags":"","title":"Retention of Tenant Data from Blocks Storage"},{"body":" Author: Daniel Blando Date: August 2022 Status: Proposed Background Cortex implements a ring structure to share information of registered pods for each service. The data stored and used by the ring need to be implemented via Codec interface. Currently, the only supported Codec to the ring is Desc. Desc is a proto.Message with a list of instances descriptions. It is used to store the data for each pod and saved on a supported KV store. Currently, Cortex supports memberlist, consul and etcd as KV stores. Memberlist works implementing a gossip protocol while consul and etcd are a KV store service.\nThe ring is used by different services using a different ring key to store and receive the values from the KV store. For example, ingester service uses the key “ingester” to save and load data from KV. As the saved data is a Desc struct only one key is used for all the information.\nProblem Each service using a single key to save and load information creates a concurrency issue when multiple pods are saving the same key. When using memberlist, the issue is mitigate as the information is owned by all pods and timestamp is used to confirm latest data. For consul and etcd, all pods compete to update the key at the same time causing an increase on latency and failures direct related to number of pods running. Cortex and etcd implementation use a version tag to make sure no data is being overridden causing the problem of write failures.\nOn a test running cortex with etcd, distributor was scaled to 300 pods and latency increase was noticed coming from etcd usage. We can also notice 5xx happening when etcd was running. 17:14 - Running memberlist, p99 around 5ms 17:25 - Running etcd, p99 around 200ms 17:25 to 17:34 migrating to multikey After running etcd multikey poc, p99 around 25ms\nProposal Multikey interface The proposal is separate the current Desc struct which contains a list of key value in multiple keys. Instead of saving one “ingester” key, the KV store will have “ingester-1”, “ingester-2” keys saved.\nCurrent:\nKey: ingester/ring/ingester Value: { \"ingesters\": { \"ingester-0\": { \"addr\": \"10.0.0.1:9095\", \"timestamp\": 1660760278, \"tokens\": [ 1, 2 ], \"zone\": \"us-west-2b\", \"registered_timestamp\": 1660708390 }, \"ingester-1\": ... } } Proposal:\nKey: ingester/ring/ingester-0 Value: { \"addr\": \"10.0.0.1:9095\", \"timestamp\": 1660760278, \"tokens\": [ 1, 15 ], \"zone\": \"us-west-2b\", \"registered_timestamp\": 1660708390 } Key: ingester/ring/ingester-1 Value: { \"addr\": \"10.0.0.2:9095\", \"timestamp\": 1660760378, \"tokens\": [ 5, 28 ], \"zone\": \"us-west-2b\", \"registered_timestamp\": 1660708572 } The proposal is to create an interface called MultiKey. The interface allows KV store to request the codec to split and join the values is separated keys.\ntype MultiKey interface { SplitById() map[string]interface{} JoinIds(map[string]interface{}) Multikey GetChildFactory() proto.Message FindDifference(MultiKey) (Multikey, []string, error) } SplitById - responsible to split the codec in multiple keys and interface. JoinIds - responsible to receive multiple keys and interface creating the codec object GetChildFactory - Allow the kv store to know how to serialize and deserialize the interface returned by “SplitById”. The interface returned by SplitById need to be a proto.Message FindDifference - optimization used to know what need to be updated or deleted from a codec. This avoids updating all keys every time the coded change. First parameter returns a subset of the Multikey to be updated. Second is a list of keys to be deleted. The codec implementation will change to support multiple keys. Currently, the codec interface for KV store supports only Encode and Decode. New methods will be added which would be used only by the KV stores implementing the multi key functionality.\ntype Codec interface { //Existen Decode([]byte) (interface{}, error) Encode(interface{}) ([]byte, error) CodecID() string //Proposed DecodeMultiKey(map[string][]byte) (interface{}, error) EncodeMultiKey(interface{}) (map[string][]byte, error) } DecodeMultiKey - called by KV store to decode data downloaded. This function will use the JoinIds method. EncodeMultiKey - called by KV store to encode data to be saved. This function will use the SplitById method. The new KV store will know the data being saved is a reference for multikey. It will use the FindDifference to know which keys need to be updated. The codec implementation for the new methods will use the JoinIds and SplitById to know how to separate the codec in multiple keys. The DecodeMultiKey will also use GetChildFactory to know how to decode the data stored in the kv store.\nExample of CAS being used with multikey design: ","categories":"","description":"","excerpt":" Author: Daniel Blando Date: August 2022 Status: Proposed Background …","ref":"/docs/proposals/ring-multikey/","tags":"","title":"Ring Multikey"},{"body":" Author: Soon-Ping Phang Date: June 2022 Status: Deprecated Introduction This proposal is deprecated in favor of the new proposal\nThis proposal consolidates multiple existing PRs from the AWS team working on this feature, as well as future work needed to complete support. The hope is that a more holistic view will make for more productive discussion and review of the individual changes, as well as provide better tracking of overall progress.\nThe original issue is #4435.\nProblem Rulers in Cortex currently run with a replication factor of 1, wherein each RuleGroup is assigned to exactly 1 ruler. This lack of redundancy creates the following risks:\nRule group evaluation Missed evaluations due to a ruler outage, possibly caused by a deployment, noisy neighbour, hardware failure, etc. Missed evaluations due to a ruler brownout due to other tenant rule groups sharing the same ruler (noisy neighbour) API -inconsistent API results during resharding (e.g. due to a deployment) when rulers are in a transition state loading rule groups This proposal attempts to mitigate the above risks by enabling a ruler replication factor of greater than 1, allowing multiple rulers to evaluate the same rule group — effectively, the ruler equivalent of ingester HA already supported in Cortex.\nProposal Make ReplicationFactor configurable ReplicationFactor in Ruler is currently hardcoded to 1. Making this a configurable parameter is the first step to enabling HA in ruler, and would also be the mechanism for the user to turn the feature on. The parameter value will be 1 by default, equating to the feature being turned off by default.\nA replication factor greater than 1 will result in the same rules being evaluated by multiple ruler instances.\nThis redundancy will allow for missed or skipped rule evaluations from single ruler outages to be covered by other instances evaluating the same rules.\nThere is also the question of duplicate metrics generated by replicated evaluation. We do not expect this to be a problem, as all replicas must use the same slotted intervals when evaluating rules, which should result in the same timestamp applying to metrics generated by each replica, with the samples being deduplicated in the ingestion path.\nPR #4712 [open]\nAdjustments for ALERTS_FOR_STATE When an alert fires in a rulegroup shared across multiple rulers, the alert on each ruler will have slightly different timestamps due to the differences in evaluation time on each ruler. This results in the ALERTS_FOR_STATE metric being deduplicated with the “duplicate sample for timestamp” error. To prevent this, we synchronize Alert.activeAt for the same alerts in different rulers using a post-processing hook.\nPRs:\nPrometheus PR #9665 [merged] Prometheus PR #10070 [merged] Cortex PR #4712 [open] Weak and Strong Quorum in ListRules and ListAlerts ListRules/ListAlerts will return inconsistent responses while a new configuration propagates to multiple ruler HA instances. For most users, this is an acceptable side-effect of an eventually consistent HA architecture. However, some use-cases have stronger consistency requirements and are willing to sacrifice some availability for those use-cases. For example, an alerts client might want to verify that a change has propagated to a majority of instances before informing the user that the update succeeded. To enable this, we propose adding an optional quorum API parameter with the following behaviour:\nquorum=weak (default)\nBiased towards availability, ListRules will perform a best-effort merge of the results from at least $i = tenantShardSize - replicationFactor + 1$ instances. If a rulegroup definition does not satisfy a quorum of $q = \\lfloor{replicationFactor / 2}\\rfloor + 1$ copies, it will choose the most recently evaluated version of that rulegroup for the final result set. An error response will be returned only if all instances return an error. Note that a side-effect of this rule is that the API will return a result even if all but one ruler instances in the tenant’s shard is unavailable. quorum=strong\nBiased towards consistency, ListRules will query at least $i = tenantShardSize - \\lfloor{replicationFactor / 2}\\rfloor + 1$ instances. If any rulegroup does not satisfy a quorum of $q = \\lfloor{replicationFactor / 2}\\rfloor + 1$ copies, a 503 error will be returned. PR #4768 [open]\nAlternatives to a quorum API parameter Weak or strong quorum by default Another option for making ListRules work in HA mode is to implement one of the quorum rules (weak or strong) as the default, with no control to select between the two, outside of maybe a configuration parameter. AWS itself runs multitenant Cortex instances, and we have an internal use-case for the strong quorum implementation, but we do not want to impose the subsequent availability hit on our customers, particularly given that replication_factor is not currently a tenant-specific parameter in Cortex, for ingesters, alert manager, or ruler.\nMaking HA availability the default, while giving users the choice to knowingly request for more consistency at the cost of more error-handling seems like a good balance.\n","categories":"","description":"","excerpt":" Author: Soon-Ping Phang Date: June 2022 Status: Deprecated …","ref":"/docs/proposals/ruler-ha/","tags":"","title":"Ruler HA"},{"body":" Author: Anand Rajagopal Date: Aug 2024 Status: Proposed Problem Rulers in Cortex currently run with a replication factor of 1, wherein each RuleGroup is assigned to exactly 1 ruler. This lack of redundancy creates the following risks:\nRule group evaluation Missed evaluations due to a ruler outage, possibly caused by a deployment, noisy neighbour, hardware failure, etc. Missed evaluations due to a ruler brownout due to other tenant rule groups sharing the same ruler (noisy neighbour) API Inconsistent API results during resharding (e.g. due to a deployment) when rulers are in a transition state loading rule groups This proposal attempts to mitigate the above risks by enabling a ruler replication factor of greater than 1, allowing multiple rulers to evaluate the same rule group — effectively.\nProposal Make ReplicationFactor configurable ReplicationFactor in Ruler is currently hardcoded to 1. Making this a configurable parameter is the first step to enabling HA in ruler. The parameter value will be 1 by default. To enable Ruler HA for rule group evaluation, a new flag will be created\nA replication factor greater than 1 will result in the following\nRing will pick R rulers for a rule group where R=RF The primary ruler (R1), when active, will take ownership of the rule group Non-primary ruler R2 will check if R1 is active. If R1 is not active, R2 will take ownership of the rule group Non-primary ruler R3 (if RF=3) will check if R1 and R2 are active. If they are both inactive/unhealthy, then R3 will take owership of the rule group Non-primary rulers will drop their ownership when R1 becomes active after an outage With this redundancy, the maximum duration of missed evaluations will be limited to the sync interval of the rule groups, reducing the impact of primary Ruler unavailability.\nPrometheus change No Prometheus change is required for this proposal\nAPI HA An interim solution is addressed in this #5773 PR. This will be modified such that the replicas will return both active and passive rule groups and the API handler will continue to de-duplicate the results. The difference is that after Ruler HA, the replicas could potentially return proper rule group state if those replicas evaluated the rule group\nPRs:\nFor Rule evaluation #6129 For API HA #5773 ","categories":"","description":"","excerpt":" Author: Anand Rajagopal Date: Aug 2024 Status: Proposed Problem …","ref":"/docs/proposals/ruler-high-availability/","tags":"","title":"Ruler High Availability"},{"body":" Author: Rees Dooley Date: November 2021 Status: Accepted Overview This document aims to describe how to implement the ability to allow rules to cover data from more than a single Cortex tenant, here after referred to as federated rules. Since currently rules are owned by, query data from and save resulting series in the same tenant, this document aims to provide clear delineation of who owns a federated rule, what tenants the federated rule queries data from and where the federated rule saves resulting series.\nA federated rule is any rule which contains the src_tenants field.\nReasoning The primary use case for allowing federated rules which query data from multiple tenants is the administration of cortex.\nIn the case of the administration of cortex, when running Cortex within a large organization, there may be metrics spanning across tenants which might be desired to be monitored e.g. administrative metrics of the cortex system like prometheus_rule_evaluation_failures_total aggregated by __tenant_id__. In this case, a team e.g. infra may wish to be able to create a rule, owned by infra which queries multiple tenants t0|t1|...|ti and stores resulting series in infra.\nChallenges Allow federated rules behind feature flag Challenge Federated tenant rules and alerts will not be a good fit for organization and should be behind a feature flag.\nProposal For federated rules, creation of federated rules (those sourcing data from multiple tenants) should be blocked behind the feature flag ruler.enable-federated-rules\nIf tenant federation is enabled, then ruler should use a mergeQueryable to aggregate the results of querying multiple tenants.\nAllow federated rules only for select tenants Challenge For many organizations, the ability for any tenant to write a rule querying any other tenant is not acceptable and more fine grained control is required\nProposal Since the current default is that a tenant should only be able to write rules against itself, we suggest a config option ruler.allowed-federated-tenants, a string slice of OrgIDs like infra or 0|1|2|3|4 which are allowed to write rules against all tenants. If a tenant bar attempts to create a federated rule, an error should be returned by the ruler api. Similarly an option ruler.disallowed-federated-tenants explicitly states a list of tenants for which federated rules are not allowed. Combining these in a util.AllowedTenants should allow one to quickly determine if federation is enabled or disabled for a given tenant at rule creation.\nWhere to store resulting series of federated rule Challenge A single tenant rule always stores produced series in the tenant where the rule exists. This 1 -\u003e 1 mapping becomes a many -\u003e 1 mapping for federated rules.\nProposal Tenants owning a federated rule the resulting series is saved in the tenant which owns the rule.\nWhich tenants to query from for federated rules Challenge A single tenant rule always queries the tenant which owns the rule. This 1 -\u003e 1 mapping becomes a 1 -\u003e many mapping for federated rules.\nProposal As some use cases will demand that a specific federated rule, querying tenant B and C, is stored in the owning teams tenant A, an option to allow explicit assignment of source tenants for a federated rule is needed.\nTo support this we suggest an additional field src_tenants on the rule group containing an array of OrgIDs e.g. [t0,t1,...,ti] which when present determines which tenants to query for the given rule. Rule group is chosen as it reduces repetition between rules.\nConclusion Challenge Status Allow federated rules behind feature flag Planned but not yet implemented Allow federated rules only for select tenants Planned but not yet implemented Where to store resulting series of federated rules Planned but not yet implemented Which tenants to query from for federated rules Planned but not yet implemented ","categories":"","description":"","excerpt":" Author: Rees Dooley Date: November 2021 Status: Accepted Overview …","ref":"/docs/proposals/ruler-tenant-federation/","tags":"","title":"Ruler Tenant Federation"},{"body":" Author: Josh Abreu Date: December 2020 Status: Accepted Context and Background The Cortex Alertmanager at its current state supports high-availability by using the same technique as the upstream Prometheus Alertmanager: we gossip silences and notifications between replicas to achieve eventual consistency. This allows us to tolerate machine failure without interruption of service. The caveat here is traffic between the ruler and Alertmanagers must not be load balanced as alerts themselves are not gossiped.\nBy itself it is not horizontally scalable; it is recommended to run a maximum of 3 replicas (for high availability). Each alertmanager replica will contain an internal instance for every tenant. The Alertmanager uses roughly ~3.7MB of memory and ~0.001 CPU cores per tenant, with an average tenant having ~10 active alerts and ~1 silences. These stats were measured from a cluster with ~80 QPS (Alerts received Per Second), ~40 NPS (Notifications Per Second), and ~700 configurations over a single replica.\nProblem and Requirements Current numbers show a reasonably sized machine can handle 2000 tenants in the current service. We would like to be able to scale this up to 10x without increasing the machine size; i.e. we would like to make Cortex Alertmanager service horizontally scalable.\nFurthermore, we desire to preserve the following characteristics:\nA single Cortex Alertmanager replica crashing or exiting abruptly should not cause externally-visible downtime or failure to deliver notifications. Users should have an eventually consistent view of all the alerts current firing and under active notification, favouring availability of the Cortex Alertmanager over consistency. We should be able to scale up, down and roll out new versions of the service without any service interruption or data loss. Design This is a big design, and as such we have divided the problem up into 4 areas: Routing \u0026 Sharding, Persistence \u0026 State, Replication \u0026 Consistency, and Architecture. The options and solutions I propose are divided along these lines to aid clarity.\nRouting \u0026 Sharding To achieve horizontal scalability, we need to distribute the workload among replicas of the service. We need to choose an appropriate field to use to distribute the workload. The field must be present on all the API requests to the Alertmanager service.\nWe propose the sharding on Tenant ID. The simplicity of this implementation, would allow us to get up and running relatively quickly whilst helping us validate assumptions. We intend to use the existing ring code to manage this. Other options such as tenant ID + receiver or Tenant ID + route are relatively complex as distributor components (in this case the Ruler) would need to be aware of Alertmanager configuration.\nPersistence \u0026 State Alertmanager is a stateful service; it stores the notification state and configured silences. By default, Alertmanager persists its state to disk every 15mins. In the horizontally scalable Alertmanager service, we need to move this state around as the number of replicas grows and shrinks. We also need to persist this state across restarts and rolling upgrades.\nWe propose making each Alertmanager replica flush the state to object storage, under its own key that’s a combination of tenant ID + replica periodically. This state on durable storage will only be used when cold-starting the cluster.\nThis mechanism covers multiple challenges (scaling up \u0026 down, rolling restarts, total cluster failure). To top this off, in the implementation, we’ll always try to request state from other replicas in the cluster before ever trying to go to object storage.\nReplication \u0026 Consistency Upstream Alertmanager replicates notification state between replicas to ensure notifications are not sent more than once. Cortex Alertmanager does the same. When we move to a model where AM instances for a given tenant only live on a subset of replicas, we have to decide how we will keep these replicas in sync.\nWe have an option of doing nothing and use the existing gossip mechanism and gossip all tenants state to all replicas. The AM Router will then drop state updates for tenants which don’t shard to a given replica. I think this will be easy to implement as it requires few changes but probably won’t scale.\nHowever, I propose we Synchronized state over gRPC. The upstream Alertmanager notification dispatcher uses a timeout-based approach for its notifications. Using the “peer order”, it’ll wait a certain amount of time before letting other replicas know if this notification succeeded or not. If it didn’t, the next replica in line will try to send the notification. We propose to communicate this via gRPC calls.\nWhile I propose the use of gRPC, is good to note that doing nothing is still a solid candidate for consideration but not having to maintain and operate two separate replication patterns feels like a win.\nArchitecture To implement the sharding strategy we need a component in charge of handling incoming alerts and API requests, distributing it to the corresponding shard.\nThe idea here is to have an “Alertmanager Distributor” as a first stop in the reception of alerts. Once alerts are received, the component is in charge of validating the alerts against the limits. Validated alerts are then sent to multiple managers in parallel.\nThe individual pieces of this component (sharding, limits) cannot be optional - the optional part of it is where we decide to run it.\nWe can either run it as a separate service or embed it. I propose we simply embed it. At its core it’ll be simpler to operate. With future work making it possible to run as a separate service so that operators can scale when/if needed.\nConclusion Under the assumption we implement the options proposed above, our architecture looks like this:\nPOST /api/v1/alerts (from the ruler) can go to any Alertmanager replica. The AM distributor uses the ring to write alerts to a quorum of AM managers (reusing the existing code). We continue to use the same in-memory data structure from the upstream Alertmanager to save alerts and notify other pieces\nGET /api/v1/alerts and /api/v1/alerts/group uses the ring to find the right alertmanager replicas for the given tenant. They read from a quorum of alertmanager replicas and return an union of the results.\nAlertmanager state is replicated between instances to keep them in sync. Where the state is replicated to to is controlled by the ring.\n","categories":"","description":"","excerpt":" Author: Josh Abreu Date: December 2020 Status: Accepted Context and …","ref":"/docs/proposals/scalable-alertmanager/","tags":"","title":"Scalable Alertmanager"},{"body":" Author: Joe Elliott Date: April 2020 Status: Proposed Overview This document aims to describe the role that the Cortex Query Frontend plays in running multitenant Cortex at scale. It also describes the challenges of horizontally scaling the query frontend component and includes several recommendations and options for creating a reliably scalable query-frontend. Finally, we conclude with a discussion of the overall philosophy of the changes and propose an alternative.\nFor the original design behind the query frontend, you should read Cortex Query Optimisations design doc from 2018-07.\nReasoning Query frontend scaling is becoming increasingly important for two primary reasons.\nThe Cortex team is working toward a scalable single binary solution. Recently the query-frontend was added to the Cortex single binary mode and, therefore, needs to seamlessly scale. Technically, nothing immediately breaks when scaling the query-frontend, but there are a number of concerns detailed in Challenges And Proposals.\nAs the query-frontend continues to support additional features it will start to become a bottleneck of the system. Current wisdom is to run very few query-frontends in order to maximize Tenancy Fairness but as more features are added scaling horizontally will become necessary.\nQuery Frontend Role Load Shedding The query frontend maintains a queue per tenant of configurable length (default 100) in which it stores a series of requests from that tenant. If this queue fills up then the frontend will return 429’s thus load shedding the rest of the system.\nThis is particularly effective due to the “pull” based model in which queriers pull requests from query frontends.\nQuery Retries The query frontend is capable of retrying a query on another querier if the first should fail due to OOM or network issues.\nSharding/Parallelization The query frontend shards requests by interval and other factors to concurrently run a single query across multiple queriers.\nQuery Alignment/Caching Queries are aligned to their own step and then stored/retrieved from cache.\nTenancy Fairness By maintaining one queue per tenant, a low demand tenant will have the same opportunity to have a query serviced as a high demand tenant. See Dilutes Tenant Fairness for additional discussion.\nFor clarity, tenancy fairness only comes into play when queries are actually being queued in the query frontend. Currently this rarely occurs, but as query sharding becomes more aggressive this may become the norm.\nChallenges And Proposals Dynamic Querier Concurrency Challenge For every query frontend the querier adds a configurable number of goroutines which are each capable of executing a query. Therefore, scaling the query frontend impacts the amount of work each individual querier is attempting to do at any given time.\nScaling up may cause a querier to attempt more work than they are configured for due to restrictions such as memory and cpu limits. Additionally, the promql engine itself is limited in the number of queries it can do as configured by the -querier.max-concurrent parameter. Attempting more queries concurrently than this value causes the queries to queue up in the querier itself.\nFor similar reasons scaling down the query frontend may cause a querier to not use its allocated memory and cpu effectively. This will lower effective resource utilization. Also, because individual queriers will be doing less work, this may cause increased queueing in the query frontends.\nProposal Currently queriers are configured to have a max parallelism per query frontend. An additional “total max concurrency” flag should be added.\nTotal Max Concurrency would then be evenly divided amongst all available query frontends. This would decouple the amount of work a querier is attempting to do with the number of query frontends that happen to exist at this moment. Consequently this would allow allocated resources (e.g. k8s cpu/memory limits) to remain balanced with the work the querier was attempting as the query frontend is scaled up or down.\nA PR has already been merged to address this.\nOverwhelming PromQL Concurrency Challenge If #frontends \u003e promql concurrency then the queriers are incapable of devoting even a single worker to each query frontend without risking queueing in the querier. Queuing in the querier is a highly undesirable state and one of the primary reasons the query frontend was originally created.\nProposal When #frontends \u003e promql concurrency then each querier will maintain exactly one connection to every frontend. As the query frontend is currently coded it will attempt to use every open GRPC connection to execute a query in the attached queriers. Therefore, in this situation where #frontends \u003e promql concurrency, the querier is exposing itself to more work then it is actually configured to perform.\nTo prevent this we will add “flow control” information to the ProcessResponse message that is used to return query results from the querier to the query frontend. In an active system this message is passed multiple times per second from the queriers to the query frontends and would be a reliable way for the frontends to track the state of queriers and balance load.\nThere are a lot of options for an exact implementation of this idea. An effective solution should be determined and chosen by modeling a set of alternatives. The details of this would be included in another design doc. A simple implementation would look something like the following:\nAdd two new fields to ProcessResponse:\nmessage ProcessResponse { httpgrpc.HTTPResponse httpResponse = 1; currentConcurrency int = 2; desiredConcurrency int = 3; } currentConcurrency - The current number of queries being executed by the querier.\ndesiredConcurrency - The total number of queries that a querier is capable of executing.\nAdd a short backoff to the main frontend processing loop. This would cause the frontend to briefly back off of any querier that was overloaded but continue to send queries to those that were capable of doing work.\nif current \u003e desired { zzz := (current - desired) * backoffDuration zzz *= 1 + rand.Float64() * .1 // jitter time.Sleep(zzz) } Passing flow control information from the querier to the frontend would also open up additional future work for more sophisticated load balancing across queriers. For example by simply comparing and choosing the least congested of two queriers we could dramatically improve how well work is distributed.\nIncreased Time To Failure Challenge Scaling the query frontend also increases the per tenant queue length by creating more queues. This could result in increased latencies where failing fast (429) would have been preferred.\nThe operator could reduce the queue length per query frontend in response to scaling out, but then they would run the risk of unnecessarily failing a request due to unbalanced distribution across query frontends. Also, shorter queues run the risk of failing to properly service heavily sharded queries.\nAnother concern is that a system with more queues will take longer to recover from an production event as it will have queued up more work.\nProposal Currently we are not proposing any changes to alleviate this concern. We believe this is solvable operationally. This can be revisited as more information is gathered.\nQuerier Discovery Lag Challenge Queriers have a configurable parameter that controls how often they refresh their query frontend list. The default value is 10 seconds. After a new query frontend is added the average querier will take 5 seconds (after DNS is updated) to become aware of it and begin requesting queries from it.\nProposal It is recommended to add a readiness/health check to the query frontend to prevent it from receiving queries while it is waiting for queriers to connect. HTTP health checks are supported by envoy, k8s, nginx, and basically any commodity load balancer. The query frontend would not indicate healthy on its health check until at least one querier had connected.\nIn a k8s environment this will require two services. One service for discovery with publishNotReadyAddresses set to true and one service for load balancing which honors the healthcheck/readiness probe. After a new query-frontend instance is created the “discovery service” would immediately have the ip of the new instance which would allow queriers to discover and attach to it. After queriers had connected it would then raise its readiness probe and appear on the “load balancing” service and begin receiving traffic.\nDilutes Tenant Fairness Challenge Given f query frontends, n tenants and an average of q queries in the frontend per tenant. The following assumes that queries are perfectly distributed across query frontends. The number of tenants per instance would be:\nThe chance that a query by a tenant with Q queries in the frontend is serviced next is:\nNote that fewer query frontends caps the impact of the number of active queries per tenant. If there is only one query frontend then the equation reduces to:\nand every tenant has an equal chance of being serviced regardless of the number of queued queries.\nAdding more query frontends favors high volume tenants by giving them more slots to be picked up by the next available querier. Fewer query frontends allows for an even playing field regardless of the number of active queries.\nFor clarity, it should be noted that tenant fairness is only impacted if queries are being queued in the frontend. Under normal operations this is currently not occurring although this may change with increased sharding.\nProposal Tenancy fairness is complex and is currently not impacting our system. Therefore we are proposing a very simple improvement to the query frontend. If/when frontend queuing becomes more common this can be revisited as we will understand the problem better.\nCurrently the query frontend picks a random tenant to service when a querier requests a new query. This can increase long tail latency if a tenant gets “unlucky” and is also exacerbated for low volume tenants by scaling the query frontend. Instead the query frontend could use a round robin approach to choose the next tenant to service. Round robin is a commonly used algorithm to increase fairness in scheduling.\nThis would be a very minor improvement, but would give some guarantees to low volume tenants that their queries would be serviced. This has been proposed in this issue.\nPros: Requires local knowledge only. Easier to implement than weighted round robin.\nCons: Improvement is minor.\nAlternatives to Round Robin\nDo Nothing\nAs is noted above tenancy fairness only comes into play when queries start queueing up in the query frontend. Internal Metrics for multi-tenant Cortex at Grafana show that this has only happened 5 times in the past week significantly enough to have been caught by Prometheus.\nRight now doing nothing is a viable option that will, almost always, fairly serve our tenants. There is, however, some concern that as sharding becomes more commonplace queueing will become more common and QOS will suffer due to reasons outlined in Dilutes Tenant Fairness.\nPros: Easy!\nCons: Nothing happens!\nWeighted Round Robin\nThe query frontends could maintain a local record of throughput or work per tenant. Tenants could then be sorted in QOS bands. In its simplest form there would be two QOS bands. The band of low volume tenants would be serviced twice for every one time the band of high volume tenants would be serviced. The full details of this approach would require a separate proposal.\nThis solution would also open up interesting future work. For instance, we could allow operators to manually configure tenants into QOS bands.\nPros: Requires local knowledge only. Can be extended later to allow tenants to be manually sorted into QOS tiers.\nCons: Improvement is better than Round Robin only. Relies on even distribution of queries across frontends. Increased complexity and difficulty in reasoning about edge cases.\nWeighted Round Robin With Gossiped Traffic\nThis approach would be equivalent to Weighted Round Robin proposed above but with tenant traffic volume gossiped between query frontends.\nPros: Benefits of Weighted Round Robin without the requirement of even query distribution. Even though it requires distributed information a failure in gossip means it gracefully degrades to Weighted Round Robin.\nCons: Requires cross instance communication. Increased complexity and difficulty in reasoning about edge cases.\nAlternative The proposals in this document have preferred augmenting existing components to make decisions with local knowledge. The unstated goal of these proposals is to build a distributed queue across a scaled query frontend that reliably and fairly serves our tenants.\nOverall, these proposals will create a robust system that is resistant to network partitions and failures of individual pieces. However, it will also create a complex system that could be difficult to reason about, contain hard to ascertain edge cases and nuanced failure modes.\nThe alternative is, instead of building a distributed queue, to add a new cortex queueing service that sits in between the frontends and the queriers. This queueing service would pull from the frontends and distribute to the queriers. It would decouple the stateful queue from the stateless elements of the query frontend and allow us to easily scale the query frontend while keeping the queue itself a singleton. In a single binary HA mode one (or few) of the replicas would be leader elected to serve this role.\nHaving a singleton queue is attractive because it is simple to reason about and gives us a single place to make fair cross tenant queueing decisions. It does, however, create a single point of failure and add another network hop to the query path.\nConclusion In this document we reviewed the reasons the frontend exists, challenges and proposals to scaling the frontend and an alternative architecture that avoids most problems but comes with its own challenges.\nChallenge Proposal Status Dynamic Querier Concurrency Add Max Total Concurrency in Querier Pull Request Overwhelming PromQL Concurrency Queriers Coordinate Concurrency with Frontends Proposed Increased Time to Failure Operational/Configuration Issue. No Changes Proposed. N/A Querier Discovery Lag Query Frontend HTTP Health Checks Pull Request Dilutes Tenant Fairness Round Robin with additional alternatives proposed Pull Request ","categories":"","description":"","excerpt":" Author: Joe Elliott Date: April 2020 Status: Proposed Overview This …","ref":"/docs/proposals/scalable-query-frontend/","tags":"","title":"Scalable Query Frontend"},{"body":" Author: @pracucci, @tomwilkie, @pstibrany Reviewers: Date: August 2020 Status: Accepted, implemented in PR #3090 Shuffle sharding and zone awareness Background Cortex shards the received series across all available ingesters. In a multi-tenant cluster, each tenant series are sharded across all ingesters. This allows to horizontally scale the series across the pool of ingesters but also suffers some issues:\nGiven every tenant writes series to all ingesters, there’s no isolation between tenants - a single misbehaving tenant can affect the whole cluster. Each ingester needs an open TSDB per tenant per ingester - which has significant memory overhead. The larger the number of tenants, the higher the TSDB memory overhead, regardless of the number of series stored in each TSDB. Similarly, the number of uploaded blocks to the storage every 2 hours is a function of the number of TSDBs open for each ingester. A cluster with a large number of small tenants will upload a very large number of blocks to the storage, each block being very small, increasing the number of API calls against the storage bucket. Cortex currently supports sharding a tenant to a subset of the ingesters on the write path PR, using a feature called “subring”. However, the current subring implementation suffers two issues:\nNo zone awareness: it doesn’t guarantee selected instances are balanced across availability zones No shuffling: the implementation is based on the hash ring and it selects N consecutive instances in the ring. This means that, instead of minimizing the likelihood that two tenants share the same instances, it emphasises it. In order to provide a good isolation between tenants, we want to minimize the chances that two tenants share the same instances. Goal The goal of this work is to fix “shuffling” and “zone-awareness” when building the subring for a given tenant, honoring the following properties:\nStability: given the same ring, the algorithm always generates the same subring for a given tenant, even across different machines Consistency: when the ring is resized, only n/m series are remapped on average (where n is the number of series and m is the number of replicas). Shuffling: probabilistically and for a large enough cluster, ensure every tenant gets a different set of instances, with a reduced number of overlapping instances between two tenants to improve failure isolation. Zone-awareness (balanced): the subring built for each tenant contains a balanced number of instances for each availability zone. Selecting the same number of instances in each zone is an important property because we want to preserve the balance of in-memory series across ingesters. Having less replicas in one zone will mean more load per node in this zone, which is something we want to avoid. Proposal This proposal is based on Amazon’s Shuffle Sharding article and the algorithm has been inspired by shuffle sharding implementation in the AWS Route53 infima library.\nGiven a tenant and a shard size S (number of instances to which tenant data/workload should be sharded to), we build a subring selecting N instances from each zone, where N = ceil(S / num of zones). The shard size S is required to be a multiple of the number of zones, in order to select an equal number of instances from each zone.\nTo do it, we treat each zone as a separate ring and select N unique instances from each zone. The instances selection process works as follow:\nGenerate a seed based on the tenant ID Initialise a pseudo random number generator with the tenant’s seed. The random generator must guarantee predictable numbers given the same input seed. Generate a sequence of N random numbers, where N is the number of instances to select from the zone. Each random number is used as a “token” to look up instances in the ring. For each random number: Lookup the instance holding that token in the ring If the instance has not been previously selected, then pick it If the instance was previously selected (we call this a “collision”), then continue walking the ring clockwise until we find an instance which has not been selected yet Guaranteed properties Stability The same tenant ID always generates the same seed. Given the same seed, the pseudo number random generator always generates the same sequence of numbers.\nThis guarantees that, given the same ring, we generate the same exact subring for a given tenant.\nConsistency The consistency property is honored by two aspects of the algorithm:\nThe quantity of random numbers generated is always equal to the shard size S, even in case of “collisions”. A collision is when the instance holding the random token has already been picked and we need to select a different instance which has not been picked yet. In case of collisions, we select the “next” instance continuing walking the ring instead of generating another random number Example adding an instance to the ring Let’s consider an initial ring with 3 instances and 1 zone (for simplicity):\nI1 - Tokens: 1, 8, 15 I2 - Tokens: 5, 11, 19 I3 - Tokens: 7, 13, 21 With a replication factor = 2, the random sequence looks up:\n3 (I2) 6 (I1) Then we add a new instance and the updated ring is:\nI1 - Tokens: 1, 8, 15 I2 - Tokens: 5, 11, 19 I3 - Tokens: 7, 13, 21 I4 - Tokens: 4, 7, 17 Now, let’s compare two different algorithms to solve collisions:\nUsing the random generator:\nRandom sequence = 3 (I4), 6 (I4 - collision), 12 (I3)\nall instances are different (I4, I3) Walking the ring:\nRandom sequence = 3 (I4), 6 (I4 - collision, next is I1)\nonly 1 instance is different (I4, I1) Shuffling Unless when resolving collisions, the algorithm doesn’t walk the ring to find the next instances, but uses a sequence of random numbers. This guarantees instances are shuffled, between different tenants, when building the subring.\nZone-awareness We treat each zone as a separate ring and select an equal number of instances from each zone. This guarantees a fair balance of instances between zones.\nProof of concept We’ve built a reference implementation of the proposed algorithm, to test the properties described above.\nIn particular, we’ve observed that the actual distribution of matching instances between different tenants is very close to the theoretical one, as well as consistency and stability properties are both honored.\n","categories":"","description":"","excerpt":" Author: @pracucci, @tomwilkie, @pstibrany Reviewers: Date: August …","ref":"/docs/proposals/shuffle-sharding-and-zone-awareness/","tags":"","title":"Shuffle sharding and zone awareness"},{"body":" Author: @pracucci, @tomwilkie, @pstibrany Reviewers: Date: August 2020 Status: Proposed, partially implemented Background Cortex currently supports sharding of tenants to a subset of the ingesters on the write path PR.\nThis feature is called “subring”, because it computes a subset of nodes registered to the hash ring. The aim of this feature is to improve isolation between tenants and reduce the number of tenants impacted by an outage.\nThis approach is similar to the techniques described in Amazon’s Shuffle Sharding article, but currently suffers from a non random selection of nodes (proposed solution below).\nCortex can be configured with a default subring size, and then it can be customized on a per-tenant basis. The per-tenant configuration is live reloaded during runtime and applied without restarting the Cortex process.\nThe subring sharding currently supports only the write-path. The read-path is not shuffle sharding aware. For example, an outage of more than one ingester with RF=3 will affect all tenants, or a particularly noisy tenant wrt queries has the ability to affect all tenants.\nGoals The Cortex read path should support shuffle sharding to isolate the impact of an outage in the cluster. The shard size must be dynamically configurable on a per-tenant basis during runtime.\nThis deliverable involves introducing shuffle sharding in:\nQuery-frontend → Querier (for queries sharding) PR #3113 Querier → Store-gateway (for blocks sharding) PR #3069 Querier→ Ingesters (for queries on recent data) Ruler (for rule and alert evaluation) Prerequisite: fix subring shuffling The solution is implemented in https://github.com/cortexproject/cortex/pull/3090.\nThe problem The subring is a subset of nodes that should be used for a specific tenant.\nThe current subring implementation doesn’t shuffle tenants across nodes. Given a tenant ID, it finds the first node owning the hash(tenant ID) token and then it picks N distinct consecutive nodes walking the ring clockwise.\nFor example, in a cluster with 6 nodes (numbered 1-6) and a replication factor of 3, three tenants (A, B, C) could have the following shards:\nTenant ID Node 1 Node 2 Node 3 Node 4 Node 5 Node 6 A x x x B x x x C x x x Proposal We propose to build the subring picking N distinct and random nodes registered in the ring, using the following algorithm:\nSID = tenant ID SID = hash(SID) Look for the node owning the token range containing FNV-1a(SID) Loop to (2) until we’ve found N distinct nodes (where N is the shard size) hash() function to be decided. The required property is to be strong enough to not generate loops across multiple subsequent hashing of the previous hash.\nQuery-frontend → Queriers shuffle sharding Implemented in https://github.com/cortexproject/cortex/pull/3113.\nHow querier runs query-frontend jobs Today each querier connects to each query-frontend instance, and calls a single “Process” method via gRPC.\n“Process” is a bi-directional streaming gRPC method – using the server-to-client stream for sending requests from query-frontend to the querier, and client-to-server stream for returning results from querier to the query-frontend. NB this is the opposite of what might be considered normal. Query-frontend scans all its queues with pending query requests, and picks a query to execute based on a fair schedule between tenants.\nThe query request is then sent to an idle querier worker over the stream opened in the Process method, and the query-frontend then waits for a response from querier. This loop repeats until querier disconnects.\nProposal To support shuffle sharding, Query-Frontends will keep a list of connected Queriers, and randomly (but consistently between query-frontends) choose N of them to distribute requests to. When Query-Frontend looks for the next request to send to a given querier, it will only consider tenants that “belong” to the Querier.\nTo choose N Queriers for a tenant, we propose to use a simple algorithm:\nSort all Queriers by their ID SID = tenant ID SID = hash(SID) Pick the querier from the list of sorted queries with:\nindex = FNV-1a(SID) % number of Queriers Loop to (3) until we’ve found N distinct queriers (where N is the shard size) and stop early if there aren’t enough queriers hash() function to be decided. The required property is to be strong enough to not generate loops across multiple subsequent hashing of the previous hash.\nProperties Stability: this will produce the same result on all query-frontends as long as all queriers are connected to all query-frontends. Simplicity: no external dependencies. No consistent hashing: adding/removing queriers will cause “resharding” of tenants between queriers. While in general that’s not desirable property, queriers are stateless so it doesn’t seem to matter in this case. Implementation notes Caching: once this list of queriers to use for a tenant is computed in the query-frontend, it is cached in memory until queriers are added or removed. Per-tenant cache entries will have a TTL to discard tenants not “seen” since a while. Querier ID: Query-frontends currently don’t have any identity for queriers. We need to introduce sending of a unique ID (eg. hostname) by querier to query-frontend when it calls “Process” method. Backward-compatibility: when querier shuffle sharding is enabled, the system expects that both query-frontend and querier will run a compatible version. Cluster version upgrade will require to rollout new query-frontends and queriers first, and then enable shuffle sharding. UI: we propose to expose the current state of the query-frontend through a new endpoint which should display: Which querier are connected to the query-frontend Are there any “old” queriers, that are receiving requests from all tenants? Mapping of tenants to queriers. Note that this mapping may only be available for tenants with pending requests on given query-frontend, and therefore be very dynamic. Configuration Shard size will be configurable on a per-tenant basis via existing “runtime-configuration” mechanism (limits overrides). Changing a value for a tenant needs to invalidate cached per-tenant queriers. Queriers shard size will be a different setting than then one used for writes. Evaluated alternatives Use the subring An alternative option would be using the subring. This implies having queriers registering to the hash ring and query-frontend instances using the ring client to find the queriers subring for each tenant.\nThis solution looks adding more complexity without any actual benefit.\nChange query-frontend → querier architecture Completely different approach would be to introduce a place where starting queriers would register (eg. DNS-based service discovery), and let query-frontends discover queriers from this central registry.\nPossible benefit would be that queriers don’t need to initiate connection to all query-frontends, but query-frontends would only connect to queriers for which they have actual pending requests. However this would be a significant redesign of how query-frontend / querier communication works.\nQuerier → Store-gateway shuffle sharding Implemented in https://github.com/cortexproject/cortex/pull/3069.\nIntroduction As of today, the store-gateway supports blocks sharding with customizable replication factor (defaults to 3). Blocks of a single tenant are sharded across all store-gateway instances and so to execute a query the querier may touch any store-gateway in the cluster.\nThe current sharding implementation is based on a hash ring formed by store-gateway instances.\nProposal The proposed solution to add shuffle sharding support to the store-gateway is to leverage on the existing hash ring to build a per-tenant subring, which is then used both by the querier and store-gateway to know to which store-gateway a block belongs to.\nConfiguration Shuffle sharding can be enabled in the store-gateway configuration. It supports a default sharding factor, which is overridable on a per-tenant basis and live reloaded during runtime (using the existing limits config). The querier already requires the store-gateway configuration when the blocks sharding is enabled. Similarly, when shuffle sharding is enabled the querier will require the store-gateway shuffle sharding configuration as well. Implementation notes When shuffle sharding is enabled:\nThe store-gateway syncUsersBlocks() will build a tenant’s subring for each tenant found scanning the bucket and will skip any tenant not belonging to its shard.\nLikewise, ShardingMetadataFilter will first build a tenant’s subring and then will use the existing logic to filter out blocks not belonging to store-gateway instance itself. The tenant ID can be read from the block’s meta.json. The querier blocksStoreReplicationSet.GetClientsFor() will first build a tenant’s subring and then will use the existing logic to find out to which store-gateway instance each requested block belongs to. Evaluated alternatives Given the store-gateways already form a ring and building the shuffle sharding based on the ring (like in the write path) doesn’t introduce extra operational complexity, we haven’t discussed alternatives.\nQuerier→ Ingesters shuffle sharding We’re currently discussing/evaluating different options.\nProblem Cortex must guarantee query correctness; transiently incorrect results may be cached and returned forever. The main problem to solve when introducing ingesters shuffle sharding on the read path is to make sure that a querier fetch data from all ingesters having at least 1 sample for a given tenant.\nThe problem to solve is: how can a querier efficiently find which ingesters have data for a given tenant? Each option must consider the changing of the set of ingesters and the changing of each tenant’s subring size.\nProposal: use only the information contained in the ring. This section describes an alternative approach. Discussion is still on-going.\nThe idea is for the queries to be able to deduce what ingesters could possibly hold data for a given tenant by just consulting the ring (and the per-tenant sub ring sizes). We posit that this is possible with only a single piece of extra information: a single timestamp per ingester saying when the ingester first joined the ring.\nScenario: ingester scale up When a new ingester is added to the ring, there will be a set of user subrings that see a change: an ingester being removed, and a new one being added. We need to guarantee that for some time period (the block flush interval), the ingester removed is also consulted for queries.\nTo do this, during the subring selection if we encounters an ingester added within the time period, we will add this to the subring but continue node selection as before - in effect, selecting an extra ingester:\nvar ( subringSize int selectedNodes []Node deadline = time.Now().Add(-flushWindow) ) for len(selectedNodes) \u003c subringSize { token := random.Next() node := getNodeByToken(token) for { if node in selectedNodes { node = node.Next() continue } if node.Added.After(deadline) { subringSize++ selectedNodes.Add(node) node = node.Next() continue ) selectedNodes.Add(node) break ) } Scenario: ingester scale down When an ingester is permanently removed from the ring it will flush its data to the object store and the subrings containing the removed ingester will gain a “new” ingester. Queries consult the store and merge the results with those from the ingesters, so no data will be missed.\nQueriers and store-gateways will discover newly flushed blocks on next sync (-blocks-storage.bucket-store.sync-interval, default 5 minutes). Multiple ingesters should not be scaled-down within this interval.\nTo improve read-performance, queriers and rulers are usually configured with non-zero value of -querier.query-store-after option. This option makes queriers and rulers to consult only ingesters when running queries within specified time window (eg. 12h). During scale-down this needs to be lowered in order to let queriers and rulers use flushed blocks from the storage.\nScenario: increase size of a tenant’s subring Node selection for subrings is stable - increasing the size of a subring is guaranteed to only add new nodes to it (and not remove any nodes). Hence, if a tenant’s subring is increase in size the queriers will notice the config change and start consulting the new ingester.\nScenario: decreasing size of a tenant’s subring If a tenant’s subring decreases in size, there is currently no way for the queriers to know how big the ring was previously, and hence they will potentially miss an ingester with data for that tenant.\nThis is deemed an infrequent operation that we considered banning, but have a proposal for how we might make it possible:\nThe proposal is to have separate read subring and write subring size in the config. The read subring will not be allowed to be smaller than the write subring. When reducing the size of a tenant’s subring, operators must first reduce the write subring, and then two hours later when the blocks have been flushed, the read subring. In the majority of cases the read subring will not need to be specified, as it will default to the write subring size.\nConsidered alternative #1: Ingesters expose list of tenants A possible solution could be keeping in the querier an in-memory data structure to map each ingester to the list of tenants for which it has some data. This data structure would be constructed at querier startup, and then periodically updated, interpolating two information:\nThe current state of the ring The list of tenants directly exposed by each ingester (via a dedicated gRPC call) Scenario: new querier starts up When a querier starts up and before getting ready:\nIt scans all ingesters (discovered via the ring) and fetches the list of tenants for which each ingester has some data For each found tenant (unique list of tenant IDs across all ingesters responses), the querier looks at the current state of the ring and adds to the map the list of ingesters currently assigned to the tenant shard, even if they don’t hold any data yet (because may start receiving series shortly) Then the querier watches the ingester ring and rebuilds the in-memory map whenever the ring topology changes.\nScenario: querier receives a query for an unknown tenant A new tenant starts remote writing to the cluster. The querier doesn’t know it in its in-memory map, so it adds the tenant on the fly to the map just looking at the current state of the ring.\nScenario: ingester scale up / down When a new ingester is added / removed to / from the ring, the ring topology changes and queriers will update the in-memory map.\nScenario: per-tenant shard size increases Queriers periodically (every 1m) reload the limits config file. When a tenant shard size change is detected, the querier updates the in-memory map for the affected tenant.\nIssue: some time series data may be missing in queries up to 1m.\nEdge case: queriers notice the ring topology change before distributors Consider the following scenario:\nTenant A shard is composed by ingesters 1,2,3,4,5,6 Tenant A is remote writing 1 single series and gets replicated to ingester 1,2,3 The ring topology changes and tenant A shard is ingesters 1,2,3,7,8,9 Querier notices the ring topology change and updates the in-memory map. Given tenant A series were only on ingester 1,2,3, the querier maps tenant A to ingester 1,2,3 (because of what received from ingesters via gRPC) and 7,8,9 (because of the current state of the ring) Distributor hasn’t updated the ring state yet Tenant A remote writes 1 new series, which get replicated to 4,5,6 Distributor updates the ring state Race condition: querier will not know that ingesters 4,5,6 contains tenant A data until the next sync Considered alternative #2: streaming updates from ingesters to queriers This section describes an alternative approach.\nCurrent state As of today, queriers discover ingesters via the ring:\nIngesters register (and update their heartbeat timestamp) to the ring and queriers watch the ring, keeping an in-memory copy of the latest ingesters ring state. Queriers use the in-memory ring state to discover all ingesters that should be queried at query time. Proposal The proposal is to expose a new gRPC endpoint on ingesters, which allows queriers to receive a stream of real time updates from ingesters about the tenants for which an ingester currently has time series data.\nFrom the querier side:\nAt startup the querier discovers all existing ingesters. For each ingester, the querier calls the ingester’s gRPC endpoint WatchTenants() (to be created). As soon as the WatchTenants() rpc is called, the ingester sends the entire set of tenants to the querier and then will send incremental updates (tenant added or removed from ingester) while the WatchTenants() stream connection is alive. If the querier loses the connection to an ingester, it will automatically retry (with backoff) while the ingester is within the ring. The querier watches the ring to discover added/removed ingesters. When an ingester is added, the querier adds the ingester to the pool of ingesters whose state should be monitored via WatchTenants(). At query time, the querier looks for all ingesters within the ring. There are two options: The querier knows the state of the ingester: the ingester will be queried only if it contains data for the query’s tenant. The querier doesn’t know the state of the ingester (eg. because it was just registered to the ring and WatchTenants() hasn’t succeeded yet): the ingester will be queried anyway (correctness first). The querier will fine tune gRPC keepalive settings to ensure a lost connection between the querier and ingester will be early detected and retried. Trade-offs Pros:\nThe querier logic, used to find ingesters for a tenant’s shard, does not require to watch the overrides config file (containing tenant shard size override). Watching the file in the querier is problematic because of introduced delays (ConfigMap update and Cortex file polling) which could lead to distributors apply changes before queriers. The querier never uses the current state of the ring as a source of information to detect which ingesters have data for a specific tenant. This information comes directly from the ingesters themselves, which makes the implementation less likely to be subject to race conditions. Cons:\nEach querier needs to open a gRPC connection to each ingester. Given gRPC supports multiplexing, the underlying TCP connection could be the same connection used to fetch samples from ingesters at query time, basically having 1 single TCP connection between a querier and an ingester. The “Edge case: queriers notice the ring topology change before distributors” described in attempt #1 can still happen in case of delays in the propagation of the state update from an ingester to queriers: Short delay: a short delay (few seconds) shouldn’t be a real problem. From the final user perspective, there’s no real difference between this edge case and a delay of few seconds in the ingestion path (eg. Prometheus remote write lagging behind few seconds). In the real case of Prometheus remote writing to Cortex, there’s no easy way to know if the latest samples are missing because has not been remote written yet by Prometheus or any delay in the propagation of this information between ingesters and queriers. Long delay: in case of networking issue propagating the state update from an ingester to the querier, the gRPC keepalive will trigger (because of failed ping-pong) and the querier will remove the failing ingesters in-memory data, so the ingester will be always tried by the querier for any query, until the state update will be re-established. Ruler sharding Introduction The ruler currently supports rule groups sharding across a pool of rulers. When sharding is enabled, rulers form a hash ring and each ruler uses the ring to check if it should evaluate a specific rule group.\nAt a polling interval (defaults to 1 minute), the ruler:\nList all the bucket objects to find all rule groups (listing is done specifying an empty delimiter so it return objects at any depth) For each discovered rule group, the ruler hashes the object key and checks if it belongs to the range of tokens assigned to the ruler itself. If not, the rule group is discarded, otherwise it’s kept for evaluation. Proposal We propose to introduce shuffle sharding in the ruler as well, leveraging on the already existing hash ring used by the current sharding implementation.\nThe configuration will be extended to allow to configure:\nEnable/disable shuffle sharding Default shard size Per-tenant overrides (reloaded at runtime) When shuffle sharding is enabled:\nThe ruler lists (ListBucketV2) the tenants for which rule groups are stored in the bucket The ruler filters out tenants not belonging to its shard For each tenant belonging to its shard, the ruler does a ListBucketV2 call with the “/” prefix and with empty delimiter to find all the rule groups, which are then evaluated in the ruler The ruler re-syncs the rule groups from the bucket whenever one of the following conditions happen:\nPeriodic interval (configurable) Ring topology changes The configured shard size of a tenant has changed Other notes The “subring” implementation is unoptimized. We will optimize it as part of this work to make sure no performance degradation is introduced when using the subring vs the normal ring. ","categories":"","description":"","excerpt":" Author: @pracucci, @tomwilkie, @pstibrany Reviewers: Date: August …","ref":"/docs/proposals/shuffle-sharding-on-the-read-path/","tags":"","title":"Shuffle sharding on the read path"},{"body":" Author: @gotjosh Reviewers: @gouthamve, @pracucci Date: March 2020 Status: Accepted Problem Statement Prometheus holds metric metadata alongside the contents of a scrape. This metadata (HELP, TYPE, UNIT and METRIC_NAME) enables some Prometheus API endpoints to output the metadata for integrations (e.g. Grafana) to consume it.\nAt the moment of writing, Cortex does not support the api/v1/metadata endpoint that Prometheus implements as metadata was never propagated via remote write. Recent work is done in Prometheus enables the propagation of metadata.\nWith this in place, remote write integrations such as Cortex can now receive this data and implement the API endpoint. This results in Cortex users being able to enjoy a tiny bit more insight on their metrics.\nPotential Solutions Before we delve into the solutions, let’s set a baseline about how the data is received. This applies almost equally for the two.\nMetadata from Prometheus is sent in the same WriteRequest proto message that the samples use. It is part of a different field (#3 given #2 is already used interally), the data is a set identified by the metric name - that means it is aggregated across targets, and is sent all at once. Implying, Cortex will receive a single WriteRequest containing a set of the metadata for that instance at an specified interval.\n. It is also important to note that this current process is an intermediary step. Eventually, metadata in a request will be sent alongside samples and only for those included. The solutions proposed, take this nuance into account to avoid coupling between the current and future state of Prometheus, and hopefully do something now that also works for the future.\nAs a reference, these are some key numbers regarding the size (and send timings) of the data at hand from our clusters at Grafana Labs:\nOn average, metadata (a combination of HELP, TYPE, UNIT and METRIC_NAME) is ~55 bytes uncompressed. at GL, on an instance with about 2.6M active series, we hold ~1241 unique metrics in total. with that, we can assume that on a worst-case scenario the metadata set for that instance is ~68 kilobytes uncompressed. by default, this data is only propagated once every minute (aligning with the default scrape interval), but this can be adjusted. Finally, what this gives us is a baseline worst-case scenario formula for the data to store per tenant: ~68KB * Replication Factor * # of Instances. Keeping in mind that typically, there’s a very high overlap of metadata across instances, and we plan to deduplicate in the ingesters. Write Path Store the metadata directly from the distributors into a cache (e.g. Memcached) Since metadata is received all at once, we could directly store into an external cache using the tenant ID as a key, and still, avoid a read-modify-write. However, a very common use case of Cortex is to have multiple Prometheus sending data for the same tenant ID. This complicates things, as it adds a need to have an intermediary merging phase and thus making a read-modify-write inevitable.\nKeep metadata in memory within the ingesters Similarly to what we do with sample data, we can keep the metadata in-memory in the ingesters and apply similar semantics. I propose to use the tenant ID as a hash key, distribute it to the ingesters (taking into account the replication factor), using a hash map to keep a set of the metadata across all instances for a single tenant, and implement a configurable time-based purge process to deal with metadata churn. Given, we need to ensure fair-use we also propose implementing limits for both the number of metadata entries we can receive and the size of a single entry.\nRead Path In my eyes, the read path seems to only have one option. At the moment of writing, Cortex uses a DummyTargetRetriever as a way to signal that these API endpoints are not implemented. We’d need to modify the Prometheus interface to support a Context and extract the tenant ID from there. Then, use the tenant ID to query the ingesters for the data, deduplicate it and serve it.\nConclusions I conclude that solution #2 is ideal for this work on the write path. It allows us to use similar semantics to samples, thus reducing operational complexity, and lays a groundwork for when we start receiving metadata alongside samples.\nThere’s one last piece to address: Allowing metadata to survive rolling restarts. Option #1 handles this well, given the aim would be to use an external cache such as Memcached. Option #2 lacks this, as it does not include any plans to persist this data. Given Prometheus (by default) sends metadata every minute, and we don’t need a high level of consistency. We expect that an eventual consistency of up to 1 minute on the default case is deemed acceptable.\nReferences Prometheus Propagate metadata via Remote Write Design Doc Prometheus Propagate metadata via Remote Write Design Issue ","categories":"","description":"","excerpt":" Author: @gotjosh Reviewers: @gouthamve, @pracucci Date: March 2020 …","ref":"/docs/proposals/support-metadata-api/","tags":"","title":"Support metadata API"},{"body":" Author: Ilan Gofman Date: June 2021 Status: Proposal Problem Currently, Cortex only implements a time series deletion API for chunk storage. We present a design for implementing time series deletion with block storage. We would like to have the same API for deleting series as currently implemented in Prometheus and in Cortex with chunk storage.\nThis can be very important for users to have as confidential or accidental data might have been incorrectly pushed and needs to be removed. As well as potentially removing high cardinality data that is causing inefficient queries.\nRelated works As previously mentioned, the deletion feature is already implemented with chunk storage. The main functionality is implemented through the purger service. It accepts requests for deletion and processes them. At first, when a deletion request is made, a tombstone is created. This is used to filter out the data for queries. After some time, a deletion plan is executed where the data is permanently removed from chunk storage.\nCan find more info here:\nCortex documentation for chunk store deletion Chunk deletion proposal Background on current storage With a block-storage configuration, Cortex stores data that could be potentially deleted by a user in:\nObject store (GCS, S3, etc..) for long term storage of blocks Ingesters for more recent data that should be eventually transferred to the object store Cache Index cache Metadata cache Chunks cache (stores the potentially to be deleted data) Query results cache (stores the potentially to be deleted data) Compactor during the compaction process Store-gateway Proposal The deletion will not happen right away. Initially, the data will be filtered out from queries using tombstones and will be deleted afterward. This will allow the user some time to cancel the delete request.\nAPI Endpoints The existing purger service will be used to process the incoming requests for deletion. The API will follow the same structure as the chunk storage endpoints for deletion, which is also based on the Prometheus deletion API.\nThis will enable the following endpoints for Cortex when using block storage:\nPOST /api/v1/admin/tsdb/delete_series - Accepts Prometheus style delete request for deleting series.\nParameters:\nstart=\u003crfc3339 | unix_timestamp\u003e Optional. If not provided, will be set to minimum possible time. end=\u003crfc3339 | unix_timestamp\u003e Optional. If not provided, will be set to maximum possible time (time when request was made). End time cannot be greater than the current UTC time. match[]=\u003cseries_selector\u003e Cannot be empty, must contain at least one label matcher argument. POST /api/v1/admin/tsdb/cancel_delete_request - To cancel a request if it has not been processed yet for permanent deletion. This can only be done before the -purger.delete-request-cancel-period has passed. Parameters:\nrequest_id GET /api/v1/admin/tsdb/delete_series - Get all delete requests id’s and their current status.\nPrometheus also implements a clean_tombstones API which is not included in this proposal. The tombstones will be deleted automatically once the permanent deletion has taken place which is described in the section below. By default, this should take approximately 24 hours.\nDeletion Lifecycle The deletion request lifecycle can follow these 3 states:\nPending - Tombstone file is created. During this state, the queriers will be performing query time filtering. The initial time period configured by -purger.delete-request-cancel-period, no data will be deleted. Once this period is over, permanent deletion processing will begin and the request is no longer cancellable. Processed - All requested data has been deleted. Initially, will still need to do query time filtering while waiting for the bucket index and store-gateway to pick up the new blocks. Once that period has passed, will no longer require any query time filtering. Deleted - The deletion request was cancelled. A grace period configured by -purger.delete-request-cancel-period will allow the user some time to cancel the deletion request if it was made by mistake. The request is no longer cancelable after this period has passed. Filtering data during queries while not yet deleted: Once a deletion request is received, a tombstone entry will be created. The object store such as S3, GCS, Azure storage, can be used to store all the deletion requests. See the section below for more detail on how the tombstones will be stored. Using the tombstones, the querier will be able to filter the to-be-deleted data initially. If a cancel delete request is made, then the tombstone file will be deleted. In addition, the existing cache will be invalidated using cache generation numbers, which are described in the later sections.\nThe compactor’s BlocksCleaner service will scan for new tombstone files and will update the bucket-index with the tombstone information regarding the deletion requests. This will enable the querier to periodically check the bucket index if there are any new tombstone files that are required to be used for filtering. One drawback of this approach is the time it could take to start filtering the data. Since the compactor will update the bucket index with the new tombstones every -compactor.cleanup-interval (default 15 min). Then the cached bucket index is refreshed in the querier every -blocks-storage.bucket-store.sync-interval (default 15 min). Potentially could take almost 30 min for queriers to start filtering deleted data when using the default values. If the information requested for deletion is confidential/classified, the time delay is something that the user should be aware of, in addition to the time that the data has already been in Cortex.\nAn additional thing to consider is that this would mean that the bucket-index would have to be enabled for this API to work. Since the plan is to make to the bucket-index mandatory in the future for block storage, this shouldn’t be an issue.\nSimilar to the chunk storage deletion implementation, the initial filtering of the deleted data will be done inside the Querier. This will allow filtering the data read from both the store gateway and the ingester. This functionality already exists for the chunk storage implementation. By implementing it in the querier, this would mean that the ruler will be supported too (ruler internally runs the querier).\nStoring tombstones in object store The Purger will write the new tombstone entries in a separate folder called tombstones in the object store (e.g. S3 bucket) in the respective tenant folder. Each tombstone can have a separate JSON file outlining all the necessary information about the deletion request such as the parameters passed in the request, as well as some meta-data such as the creation date of the file. The name of the file can be a hash of the API parameters (start, end, markers). This way if a user calls the API twice by accident with the same parameters, it will only create one tombstone. To keep track of the request state, filename extensions can be used. This will allow the tombstone files to be immutable. The 3 different file extensions will be pending, processed, deleted. Each time the deletion request moves to a new state, a new file will be added with the same deletion information but a different extension to indicate the new state. The file containing the previous state will be deleted once the new one is created. If a deletion request is cancelled, then a tombstone file with the .deleted filename extension will be created.\nWhen it is determined that the request should move to the next state, then it will first write a new file containing the tombstone information to the object store. The information inside the file will be the same except the stateCreationTime, which is replaced with the current timestamp. The extension of the new file will be different to reflect the new state. If the new file is successfully written, the file with the previous state is deleted. If the write of the new file fails, then the previous file is not going to be deleted. Next time the service runs to check the state of each tombstone, it will retry creating the new file with the updated state. If the write is successful but the deletion of the old file is unsuccessful then there will be 2 tombstone files with the same filename but different extension. When BlocksCleaner writes the tombstones to the bucket index, the compactor will check for duplicate tombstone files but with different extensions. It will use the tombstone with the most recently updated state and try to delete the file with the older state. There could be a scenario where there are two files with the same request ID but different extensions: {.pending, .processed} or {.pending, .deleted}. In this case, the .processed or .deleted file will be selected as it is always the later state compared to the pending state.\nThe tombstone will be stored in a single JSON file per request and state:\n/\u003ctenantId\u003e/tombstones/\u003crequest_id\u003e.json.\u003cstate\u003e The schema of the JSON file is:\n{ \"requestId\": \u003cstring\u003e, \"startTime\": \u003cint\u003e, \"endTime\": \u003cint\u003e, \"requestCreationTime\": \u003cint\u003e, \"stateCreationTime\": \u003cint\u003e, \"matchers\": [ \"\u003cstring matcher 1\u003e\", .., \"\u003cstring matcher n\u003e\" ] }, \"userID\": \u003cstring\u003e, } Pros:\nAllows deletion and un-delete to be done in a single operation. Cons:\nNegative impact on query performance when there are active tombstones. As in the chunk storage implementation, all the series will have to be compared to the matchers contained in the active tombstone files. The impact on performance should be the same as the deletion would have with chunk storage.\nWith the default config, potential 30 minute wait for the data to begin filtering if using the default configuration.\nInvalidating cache Using block store, the different caches available are:\nIndex cache Metadata cache Chunks cache (stores the potentially to be deleted chunks of data) Query results cache (stores the potentially to be deleted data) There are two potential caches that could contain deleted data, the chunks cache, and the query results cache. Using the tombstones, the queriers filter out the data received from the ingesters and store-gateway. The cache not being processed through the querier needs to be invalidated to prevent deleted data from coming up in queries.\nFirstly, the query results cache needs to be invalidated for each new delete request or a cancellation of one. This can be accomplished by utilizing cache generation numbers. For each tenant, their cache is prefixed with a cache generation number. When the query front-end discovers a cache generation number that is greater than the previous generation number, then it knows to invalidate the query results cache. However, the cache can only be invalidated once the queriers have loaded the tombstones from the bucket index and have begun filtering the data. Otherwise, to-be deleted data might show up in queries and be cached again. One of the way to guarantee that all the queriers are using the new tombstones is to wait until the bucket index staleness period has passed from the time the tombstones have been written to the bucket index. The staleness period can be configured using the following flag: -blocks-storage.bucket-store.bucket-index.max-stale-period. We can use the bucket index staleness period as the delay to wait before the cache generation number is increased. A query will fail inside the querier, if the bucket index last update is older the staleness period. Once this period is over, all the queriers should have the updated tombstones and the query results cache can be invalidated. Here is the proposed method for accomplishing this:\nThe cache generation number will be a timestamp. The default value will be 0. The bucket index will store the cache generation number. The query front-end will periodically fetch the bucket index. Inside the compactor, the BlocksCleaner will load the tombstones from object store and update the bucket index accordingly. It will calculate the cache generation number by iterating through all the tombstones and their respective times (next bullet point) and selecting the maximum timestamp that is less than (current time minus -blocks-storage.bucket-store.bucket-index.max-stale-period). This would mean that if a deletion request is made or cancelled, the compactor will only update the cache generation number once the staleness period is over, ensuring that all queriers have the updated tombstones. For requests in a pending or processed state, the requestCreationTime will be used when comparing the maximum timestamps. If a request is in a deleted state, it will use the stateCreationTime for comparing the timestamps. This means that the cache gets invalidated only once it has been created or deleted, and the bucket index staleness period has passed. The cache will not be invalidated again when a request advances from pending to processed state. The query front-end will fetch the cache generation number from the bucket index. The query front end will compare it to the current cache generation number stored in the front-end. If the cache generation number from the front-end is less than the one from bucket index, then the cache is invalidated. In regards to the chunks cache, since it is retrieved from the store gateway and passed to the querier, it will be filtered out like the rest of the time series data in the querier using the tombstones, with the mechanism described in the previous section.\nPermanently deleting the data The proposed approach is to perform the deletions from the compactor. A new background service inside the compactor called DeletedSeriesCleaner can be created and is responsible for executing the deletion.\nProcessing This will happen after a grace period has passed once the API request has been made. By default this should be 24 hours. A background task can be created to process the permanent deletion of time series. This background task can be executed each hour.\nTo delete the data from the blocks, the same logic as the Bucket Rewrite Tool from Thanos can be leveraged. This tool does the following: tools bucket rewrite rewrites chosen blocks in the bucket, while deleting or modifying series. The tool itself is a CLI tool that we won’t be using, but instead we can utilize the logic inside it. For more information about the way this tool runs, please see the code here.\nThe compactor’s DeletedSeriesCleaner will apply this logic on individual blocks and each time it is run, it creates a new block without the data that matched the deletion request. The original individual blocks containing the data that was requested to be deleted, need to be marked for deletion by the compactor.\nWhile deleting the data permanently from the block storage, the meta.json files will be used to keep track of the deletion progress. Inside each meta.json file, we will add a new field called tombstonesFiltered. This will store an array of deletion request id’s that were used to create this block. Once the rewrite logic is applied to a block, the new block’s meta.json file will append the deletion request id(s) used for the rewrite operation inside this field. This will let the DeletedSeriesCleaner know that this block has already processed the particular deletions requests listed in this field. Assuming that the deletion requests are quite rare, the size of the meta.json files should remain small.\nThe DeletedSeriesCleaner can iterate through all the blocks that the deletion request could apply to. For each of these blocks, if the deletion request ID isn’t inside the meta.json tombstonesFiltered field, then the compactor can apply the rewrite logic to this block. If there are multiple tombstones that are currently being processing for deletions and apply to a particular block, then the DeletedSeriesCleaner will process both at the same time to prevent additional blocks from being created. If after iterating through all the blocks, it doesn’t find any such blocks requiring deletion, then the Pending state is complete and the request progresses to the Processed state.\nOne important thing to note regarding this rewrite tool is that it should not be used at the same time as when another compactor is touching a block. If the tool is run at the same time as compaction on a particular block, it can cause overlap and the data marked for deletion can already be part of the compacted block. To mitigate such issues, these are some of the proposed solutions:\nOption 1: Only apply the deletion once the blocks are in the final state of compaction.\nPros:\nSimpler implementation as everything is contained within the DeletedSeriesCleaner. Cons:\nMight have to wait for a longer period of time for the compaction to be finished. This would mean the earliest time to be able to run the deletion would be once the last time from the block_ranges in the compactor_config has passed. By default this value is 24 hours, so only once 24 hours have passed and the new compacted blocks have been created, then the rewrite can be safely run. Option 2: For blocks that still need to be compacted further after the deletion request cancel period is over, the deletion logic can be applied before the blocks are compacted. This will generate a new block which can then be used instead for compaction with other blocks.\nPros:\nThe deletion can be applied earlier than the previous options. Only applies if the deletion request cancel period is less than the last time interval for compaction is. Cons: Added coupling between the compaction and the DeletedSeriesCleaner. Might block compaction for a short time while doing the deletion. Once all the applicable blocks have been rewritten without the deleted data, the deletion request state moves to the Processed state. Once in this state, the queriers will still have to perform query time filtering using the tombstones until the old blocks that were marked for deletion are no longer queried by the queriers. This will mean that the query time filtering will last for an additional length of -compactor.deletion-delay + -compactor.cleanup-interval + -blocks-storage.bucket-store.sync-interval in the Processed state. Once that time period has passed, the queriers should no longer be querying any of the old blocks that were marked for deletion. The tombstone will no longer be used after this.\nCancelled Delete Requests If a request was successfully cancelled, then a tombstone file a .deleted extension is created. This is done to help ensure that the cache generation number is updated and the query results cache is invalidated. The compactor’s blocks cleaner can take care of cleaning up .deleted tombstones after a period of time of when they are no longer required for cache invalidation. This can be done after 10 times the bucket index max staleness time period has passed. Before removing the file from the object store, the current cache generation number must greater than or equal to when the tombstone was cancelled.\nHandling failed/unfinished delete jobs: Deletions will be completed and the tombstones will be deleted only when the DeletedSeriesCleaner iterates over all blocks that match the time interval and confirms that they have been re-written without the deleted data. Otherwise, it will keep iterating over the blocks and process the blocks that haven’t been rewritten according to the information in the meta.json file. In case of any failure that causes the deletion to stop, any unfinished deletions will be resumed once the service is restarted. If the block rewrite was not completed on a particular block, then the original block will not be marked for deletion. The compactor will continue to iterate over the blocks and process the block again.\nTenant Deletion API If a request is made to delete a tenant, then all the tombstones will be deleted for that user.\nCurrent Open Questions: If the start and end time is very far apart, it might result in a lot of the data being re-written. Since we create a new block without the deleted data and mark the old one for deletion, there may be a period of time with lots of extra blocks and space used for large deletion queries. There will be a delay between the deletion request and the deleted data being filtered during queries. In Prometheus, there is no delay. One way to filter out immediately is to load the tombstones during query time but this will cause a negative performance impact. Adding limits to the API such as: Max number of deletion requests allowed in the last 24 hours for a given tenent. Max number of pending tombstones for a given tenant. Alternatives Considered Adding a Pre-processing State The process of permanently deleting the data can be separated into 2 stages, preprocessing and processing.\nPre-processing will begin after the -purger.delete-request-cancel-period has passed since the API request has been made. The deletion request will move to a new state called BuildingPlan. The compactor will outline all the blocks that may contain data to be deleted. For each separate block that the deletion may be applicable to, the compactor will begin the process by adding a series deletion marker inside the series-deletion-marker.json file. The JSON file will contain an array of deletion request id’s that need to be applied to the block, which allows the ability to handle the situation when there are multiple tombstones that could be applicable to a particular block. Then during the processing step, instead of checking the meta.json file, we only need to check if a marker file exists with a specific deletion request id. If the marker file exists, then we apply the rewrite logic.\nAlternative Permanent Deletion Processing For processing the actual deletions, an alternative approach is not to wait until the final compaction has been completed and filter out the data during compaction. If the data is marked to be deleted, then don’t include it the new bigger block during compaction. For the remaining blocks where the data wasn’t filtered during compaction, the deletion can be done the same as in the previous section.\nPros:\nThe deletion can happen sooner. The rewrite tools creates additional blocks. By filtering the metrics during compaction, the intermediary re-written block will be avoided. Cons:\nA more complicated implementation requiring add more logic to the compactor Slower compaction if it needs to filter all the data Need to manage which blocks should be deleted with the rewrite vs which blocks already had data filtered during compaction. Would need to run the rewrite logic during and outside of compaction because some blocks that might need to be deleted are already in the final compaction state. So that would mean the deletion functionality has to be implemented in multiple places. Won’t be leveraging the rewrites tools from Thanos for all the deletion, so potentially more work is duplicated ","categories":"","description":"","excerpt":" Author: Ilan Gofman Date: June 2021 Status: Proposal Problem …","ref":"/docs/proposals/block-storage-time-series-deletion/","tags":"","title":"Time Series Deletion from Blocks Storage"},{"body":" Author: @roystchiang Reviewers: Date: August 2022 Status: Proposed Timeseries Partitioning in Compactor Introduction The compactor is a crucial component in Cortex responsible for deduplication of replicated data, and merging blocks across multiple time intervals together. This proposal will not go into great depth with why the compactor is necessary, but aims to focus on how to scale the compactor as a tenant grows within a Cortex cluster.\nProblem and Requirements Cortex introduced horizontally scaling compactor which allows multiple compactors to compact blocks for a single tenant, sharded by time interval. The compactor is capable of compacting multiple smaller blocks into a larger block, to reduce the duplicated information in index. The following is an illustration of how the shuffle sharding compactor works, where each arrow represents a single compaction that can be carried out independently. However, if the tenant is sending unique timeseries, the compaction process does not help with reducing the index size. Furthermore, this scaling of parallelism by time interval is not sufficient for a tenant with hundreds of millions of timeseries, as more timeseries means longer compaction time.\nCurrently, the compactor is bounded by the 64GB index size, and having a compaction that takes days to complete simply is not sustainable. This time includes the time to download the blocks, merging of the timeseries, writing to disk, and finally uploading to object storage.\nThe compactor is able to compact up to 400M timeseries within 12 hours, and will fail with the error of index exceeding 64GB. Depending on the number of labels and the size of labels, one might reach the 64GB limit sooner. We need a solution that is capable of:\nhandling the 64GB index limit reducing the overall compaction time downloading the data in smaller batches reducing the time required to compact Design A reminder of what a Prometheus TSDB is composed of: an index and chunks. An index is a mapping of timeseries to the chunks, so we can do a direct lookup in the chunks. Each timeseries is effectively a set of labels, mapped to a list of \u003ctimestamp, value\u003e pair. This proposal focuses on partitioning of the timeseries.\nPartitioning strategy The compactor will compact a overlapping time-range into multiple sub-blocks, instead of a single block. Cortex can determine which partition a single timeseries should go into by applying a hash to the timeseries label, and taking the modulo of the hash by the number of partition. This guarantees that with same number of partition, the same timeseries will go into the same partition.\npartitionId = Hash(timeseries label) % number of partition\nThe number of partition will be determined automatically, via a configured multiplier. This multiplier factor allows us to group just a subset of the blocks together to achieve the same deduplication factor as having all the blocks. Using a multiplier of 2 as an example, we can do grouping for partition of 2, 4 and 8. We’ll build on the actual number of partition determination in a later section.\nDetermining overlapping blocks In order to reduce the amount of time spent downloading blocks, and iterating through the index to filter out unrelated timeseries, we can do smart grouping of the blocks.\nGiven that we are always multiplying the number of partition by the multiplier factor, we can deduce from the modulo which partition could contain overlapping result\nGiven a hash N, if N % 8 == 7, then N % 4 must be 3 Given a hash N, if N % 8 == 3, then N % 4 must be 3 Given a hash N, if N % 8 == 4, then N % 4 must be 0 Given a hash N, if N % 8 == 0, then N % 4 must be 0 Hence it is safe to group blocks with N % 8 == 7 with N % 8 == 3 together with N % 4 == 3 together, and we are sure that other blocks won’t contain the same timeseries. We also know that if N % 8 == 0, then we don’t need to download blocks where N % 4 == 1 or N % 4 == 2 Given partition count and partition id, we can immediately find out which blocks are required. Using the above modulo example, we get the following partitiong mapping.\nPlanning the compaction The shuffle sharding compactor introduced additional logic to group blocks by distinct time intervals. It can also sum up the sizes of all indices to determine how many shards are required in total. Using the above example again, and assuming that each block has an index of 30GB, the sum is 30GB * 14 = 420GB, which needs to be at least 7, since maximum index size is 64GB. Using the multiplier factor, it will be rounded up to 8.\nNow the planner knows the resulting compaction will have 8 partitions, it can start planning out which groups of blocks can go into a single compaction group. Given that we need 8 partitions in total, the planner will go through the process above to find out what blocks are necessary. Using the above example again, but we have distinct time intervals, T1, T2, and T3. T1 has 2 partitions, T2 has 4 partitions, and T3 has 8 partitions, and we want to produce T1-T3 blocks Compaction Group 1-8 T1 - Partition 1-2 T2 - Partition 1-4 T3 - Partition 1-8 Compaction Group 2-8 T1 - Partition 2-2 T2 - Partition 2-4 T3 - Partition 2-8 Compaction Group 3-8 T1 - Partition 1-2 T2 - Parittion 3-4 T3 - Partition 3-8 Compaction Group 4-8 T1 - Partition 2-2 T2 - Partition 4-4 T3 - Partition 4-8 Compaction Group 5-8 T1 - Partition 1-2 T2 - Partition 1-4 T3 - Partition 5-8 Compaction Group 6-8 T1 - Partition 2-2 T2 - Partition 2-4 T3 - Partition 6-8 Compaction Group 7-8 T1 - Partition 1-2 T2 - Partition 3-4 T3 - Partition 7-8 Compaction Group 8-8 T1 - Partition 2-2 T2 - Partition 4-4 T3 - Partition 8-8 T1 - Partition 1-2 is used in multiple compaction groups, and the following section will describe how the compaction avoids duplicate timeseries in the resulting blocks\nCompaction Now that the planner has produced a compaction plan for the T1-T3 compaction groups, the compactor can start downloading the necessary blocks. Using compaction group 1-8 from above as example. T1 - Partition 1-2 was created with hash % 2 == 0, and in order to avoid having duplication information in blocks produced by compaction group 3-8, compaction group 5-8, and compaction group 7-8, we need apply the filter the %8 == 0 hash, as that’s the hash of the highest partition count.\nCompaction Workflow Compactor initializes Grouper and Planner. Compactor retrieves block’s meta.json and call Grouper to group blocks for compaction. Grouper generates partitioned compaction groups: Grouper groups source blocks into unpartitioned groups. For each unpartitioned group: Generates partitioned compaction group ID which is hash of min and max time of result block. If the ID exists under the tenant directory in block storage, continue on next unpartitioned group. Calculates number of partitions. Number of partitions indicates how many partitions one unpartitioned group would be partitioned into based on the total size of indices and number of time series from each source blocks in the unpartitioned group. Assign source blocks into each partition with partition ID (value is in range from 0 to number_of_partitions - 1). Note that one source block could be used in multiple partitions (explanation in Planning the compaction and Compaction). So multiple partition ID could be assigned to same source block. Check more partitioning examples in Compaction Partitioning Examples Generates partitioned compaction group that indicates which partition ID each blocks got assigned. Partitioned compaction group information would be stored in block storage under the tenant directory it belongs to and the stored file can be picked up by cleaner later. Partitioned compaction group information contains partitioned compaction group ID, number of partitions, list of partitions which has partition ID and list of source blocks. Store partitioned compaction group ID in block storage under each blocks’ directory that are used by the generated partitioned compaction group. Grouper returns partitioned compaction groups to Compactor. Each returned group would have partition ID, number of partitions, and list of source blocks in memory. Compactor iterates over each partitioned compaction group. For each iteration, calls Planner to make sure the group is ready for compaction. Planner collects partitioned compaction group which is ready for compaction. For each partitions in the group and for each blocks in the partition: Make sure all source blocks fit within the time range of the group. Make sure each source block with assigned partition IDs is currently not used by another ongoing compaction. This could utilize visit marker file that is introduced in #4805 by expanding it for each partition ID of the source block. If all blocks in the partition are ready to be compacted, mark status of those blocks with assigned partition ID as pending. The status information of each partition ID would be stored in block storage under the corresponding block directory in order for cleaner to pick it up later. If not all blocks in the partition are ready, continue on next partition Return all ready partitions to Compactor. Compactor starts compacting partitioned blocks. Once compaction completed, Compactor would mark status of all blocks along with assigned partition ID in the group as completed. Compactor should use partitioned compaction group ID to retrieve partitioned compaction group information from block storage to get all partition IDs assigned to each block. Then, retrieve status information of each partition ID this assigned to block under current block directory in block storage. If all assigned partition ID of the block have status set to completed, upload deletion marker for this block. Otherwise, no deletion marker would be uploaded. Clean up Workflow Cleaner would periodically check any tenants having deletion marker. If there is a deletion marker for the tenant, Cleaner should remove all blocks and then clean up other files including partitioned group information files after tenant clean up delay. If there is no deletion marker for tenant, Clean should scan any source blocks having a deletion marker. If there is a deletion marker for the block, Cleaner should delete it.\nPerformance Currently a 400M timeseries takes 12 hours to compact, without taking block download into consideration. If we have a partition count of 2, we can reduce this down to 6 hours, and a partition count of 10 is 3 hours. The scaling is not linear, and I’m still attempting to find out why. The initial result is promising enough to continue though.\nAlternatives Considered Dynamic Number of Partition We can also increase/decrease the number of partition without needing the multiplier factor. However, if a tenant is sending highly varying number of timeseries or label size, the index size can be very different, resulting in highly dynamic number of partitions. To perform deduplication, we’ll end up having to download all the sub-blocks, and it can be inefficient as less parallelization can be done, and we will spend more time downloading all the unnecessary blocks.\nConsistent Hashing Jump consistent hash, rendezvous hashing, and other consistent hashing are great algorithms to avoid reshuffling of data when introducing/removing partitions on the fly. However, it does not bring much of a benefit when determining which partition contains the same timeseries, which we need to deduplication of index.\nPartition by metric name It is likely that when a query comes, a tenant is interested in an aggregation of a single metric, across all label names. The compactor can partition by metric name, so that all timeseries with the same name will go into the same block. However, this can result in very uneven partitioning.\nArchitecture Planning The shuffle partitioning compactor introduced a planner logic, which we can extend on. This planner is responsible for grouping the blocks together by time interval, in order to compact blocks in parallel. The grouper can also determine the number of partition by looking at the sum of index file sizes. In addition, it can also do the grouping of the sub-blocks together, so we can achieve even higher parallelism.\nClean up Cortex compactor cleans up obsolete source blocks by looking at a deletion marker. The current architecture does not have the problem of having a single source block involved in multiple compaction. However, this proposal is able to achieve higher parallelism than before, hence it is possible that a source block is involved multiple times. Changes needs to be made on the compactor regarding how many plans a particular blocked is involved in, and determining when a block is safe to be deleted.\nChanges Required in Dependencies Partitioning during compaction time Prometheus exposes the possibility to pass in a custom mergeFunc. This allows us to plug in the custom partitioning strategy. However, the mergeFunc is only called when the timeseries is successfully replicated to at least 3 replicas, meaning that we’ll produce duplicate timeseries across blocks if the data is only replicated once. To work around the issue, we can propose Prometheus to allow the configuration of the MergeChunkSeriesSet.\nSource block checking Cortex uses Thanos’s compactor logic, and it has a check to make sure the source blocks of the input blocks do not overlap. Meaning that if BlockA is produced from BlockY, and BlockB is also produced from BlockY, it will halt. This is not desirable for us, since partitioning by timeseries means the same source blocks will produce multiple blocks. Reason for having this check in Thanos is supposed to prevent having duplicate chunks, but the change was introduced without knowing whether it will actually help. We’ll need to introduce a change in Thanos to disable this check, or start using Thanos compactor as a library instead of a closed box.\nWork Plan Performance test the impact on query of having multiple blocks Get real data on the efficiency of modulo operator for partitioning Get the necessary changes in Prometheus approved and merged Get the necessary changes in Thanos approved and merged Implement the number of partition determination in group Implement the grouper logic in Cortex Implement the clean up logic in Cortex Implement the partitioning strategy in Cortex, passed to Prometheus Produce the partitioned blocks Appendix Risks Is taking the modulo of the hash sufficient to produce a good distribution of partitions? What’s the effect of having too many blocks for the same time range? Frequently Asked Questions Are we able to decrease the number of partition? Using partitions of 2, and 4, and 8 as example T1 partition 1 - Hash(timeseries label) % 2 == 0 T1 partition 2 - Hash(timeseries label) % 2 == 1 T2 partition 1 - Hash(timeseries label) % 4 == 0 T2 partition 2 - Hash(timeseries label) % 4 == 1 T2 partition 3 - Hash(timeseries label) % 4 == 2 T2 partition 4 - Hash(timeseries label) % 4 == 3 T3 partition 1 - Hash(timeseries label) % 8 == 0 T3 partition 2 - Hash(timeseries label) % 8 == 1 T3 partition 3 - Hash(timeseries label) % 8 == 2 T3 partition 4 - Hash(timeseries label) % 8 == 3 T3 partition 5 - Hash(timeseries label) % 8 == 4 T3 partition 6 - Hash(timeseries label) % 8 == 5 T3 partition 7 - Hash(timeseries label) % 8 == 6 T3 partition 8 - Hash(timeseries label) % 8 == 7 We are free to produce a resulting timerange T1-T3, without having to download all 14 blocks in a single compactor If T1-T3 can fit inside 4 partitions, we can do the following grouping T1 partition 1 - Hash(timeseries label) % 2 == 0 \u0026\u0026 % 4 == 0 T2 partition 1 - Hash(timeseries label) % 4 == 0 \u0026\u0026 T3 partition 1 - Hash(timeseries label) % 8 == 0 T3 partition 5 - Hash(timeseries label) % 8 == 4 T1 partition 2 - Hash(timeseries label) % 2 == 1 \u0026\u0026 % 4 == 01 T2 partition 2 - Hash(timeseries label) % 4 == 1 T3 partition 2 - Hash(timeseries label) % 8 == 1 T3 partition 7 - Hash(timeseries label) % 8 == 5 T1 partition 1 - Hash(timeseries label) % 2 == 0 \u0026\u0026 % 4 == 2 T2 partition 3 - Hash(timeseries label) % 4 == 2 T3 partition 3 - Hash(timeseries label) % 8 == 2 T3 partition 7 - Hash(timeseries label) % 8 == 6 T1 partition 2 - Hash(timeseries label) % 2 == 1 \u0026\u0026 % 4 == 3 T2 partition 4 - Hash(timeseries label) % 4 == 3 T3 partition 4 - Hash(timeseries label) % 8 == 3 T3 partition 8 - Hash(timeseries label) % 8 == 7 If T1-T3 can fit inside 16 partitions, we can do the same grouping, and hash on top T1 partition 1 - Hash(timeseries label) % 2 == 0 \u0026\u0026 % 8 == 0 T2 partition 1 - Hash(timeseries label) % 4 == 0 \u0026\u0026 % 8 == 0 T3 partition 1 - Hash(timeseries label) % 8 == 0 T1 partition 2 - Hash(timeseries label) % 2 == 1 \u0026\u0026 % 8 == 1 T2 partition 2 - Hash(timeseries label) % 4 == 1 \u0026\u0026 % 8 == 1 T3 partition 2 - Hash(timeseries label) % 8 == 1 T1 partition 1 - Hash(timeseries label) % 2 == 0 \u0026\u0026 % 8 == 2 T2 partition 3 - Hash(timeseries label) % 4 == 2 \u0026\u0026 % 8 == 2 T3 partition 3 - Hash(timeseries label) % 8 == 2 T1 partition 2 - Hash(timeseries label) % 2 == 1 \u0026\u0026 % 8 == 3 T2 partition 4 - Hash(timeseries label) % 4 == 3 \u0026\u0026 % 8 == 3 T3 partition 4 - Hash(timeseries label) % 8 == 3 T1 partition 1 - Hash(timeseries label) % 2 == 0 \u0026\u0026 % 8 == 4 T2 partition 1 - Hash(timeseries label) % 4 == 0 \u0026\u0026 % 8 == 4 T3 partition 5 - Hash(timeseries label) % 8 == 4 T1 partition 2 - Hash(timeseries label) % 2 == 1 \u0026\u0026 % 8 == 5 T2 partition 2 - Hash(timeseries label) % 4 == 1 \u0026\u0026 % 8 == 5 T3 partition 6 - Hash(timeseries label) % 8 == 5 T1 partition 1 - Hash(timeseries label) % 2 == 0 \u0026\u0026 % 8 == 6 T2 partition 3 - Hash(timeseries label) % 4 == 2 \u0026\u0026 % 8 == 6 T3 partition 7 - Hash(timeseries label) % 8 == 6 T1 partition 2 - Hash(timeseries label) % 2 == 1 \u0026\u0026 % 8 == 7 T2 partition 4 - Hash(timeseries label) % 4 == 3 \u0026\u0026 % 8 == 7 T3 partition 8 - Hash(timeseries label) % 8 == 7 Compaction Partitioning Examples Scenario: All source blocks were compacted by partitioning compaction (Idea case) All source blocks were previously compacted through partitioning compaction. In this case for each time range, the number of blocks belong to same time range would be 2^x if multiplier is set to 2.\nTime ranges: T1, T2, T3 Source blocks: T1: B1, B2 T2: B3, B4, B5, B6 T3: B7, B8, B9, B10, B11, B12, B13, B14 Total indices size of all source blocks: 200G Number of Partitions = (200G / 64G = 3.125) =\u003e round up to next 2^x = 4\nPartitioning:\nFor T1, there are only 2 blocks which is \u003c 4. So B1 (index 0 in the time range) can be grouped with other blocks having N % 4 == 0 or 2. Because 0 % 2 == 0. B2 (index 1 in the time range) can be grouped with other blocks having N % 4 == 1 or 3. Because 1 % 2 == 1. For T2, B3 (index 0 in the time range) can be grouped with other blocks having N % 4 == 0. B4 (index 1 in the time range) can be grouped with other blocks having N % 4 == 1. B5 (index 2 in the time range) can be grouped with other blocks having N % 4 == 2. B6 (index 3 in the time range) can be grouped with other blocks having N % 4 == 3. For T3, B7 (index 0 in the time range) can be grouped with other blocks having N % 4 == 0. B8 (index 1 in the time range) can be grouped with other blocks having N % 4 == 1. B9 (index 2 in the time range) can be grouped with other blocks having N % 4 == 2. B10 (index 3 in the time range) can be grouped with other blocks having N % 4 == 3. B11 (index 4 in the time range) can be grouped with other blocks having N % 4 == 0. B12 (index 5 in the time range) can be grouped with other blocks having N % 4 == 1. B13 (index 6 in the time range) can be grouped with other blocks having N % 4 == 2. B14 (index 7 in the time range) can be grouped with other blocks having N % 4 == 3. Partitions in Partitioned Compaction Group:\nPartition ID: 0 Number of Partitions: 4 Blocks: B1, B3, B7, B11 Partition ID: 1 Number of Partitions: 4 Blocks: B2, B4, B8, B12 Partition ID: 2 Number of Partitions: 4 Blocks: B1, B5, B9, B13 Partition ID: 3 Number of Partitions: 4 Blocks: B2, B6, B10, B14 Scenario: All source blocks are level 1 blocks All source blocks are level 1 blocks. Since number of level 1 blocks in one time range is not guaranteed to be 2^x, all blocks need to be included in each partition.\nTime ranges: T1 Source blocks: T1: B1, B2, B3 Total indices size of all source blocks: 100G Number of Partitions = (100G / 64G = 1.5625) =\u003e round up to next 2^x = 2\nPartitioning: There is only one time range from all source blocks which means it is compacting level 1 blocks. Partitioning needs to include all source blocks in each partition.\nPartitions in Partitioned Compaction Group:\nPartition ID: 0 Number of Partitions: 2 Blocks: B1, B2, B3 Partition ID: 1 Number of Partitions: 2 Blocks: B1, B2, B3 Scenario: All source blocks are with compaction level \u003e 1 and were generated by compactor without partitioning compaction If source block was generated by compactor without partitioning compaction, there should be only one block per time range. Since there is only one block in one time range, that one block would be included in all partitions.\nTime ranges: T1, T2, T3 Source blocks: T1: B1 T2: B2 T3: B3 Total indices size of all source blocks: 100G Number of Partitions = (100G / 64G = 1.5625) =\u003e round up to next 2^x = 2\nPartitioning:\nFor T1, there is only one source block. Include B1 in all partitions. For T2, there is only one source block. Include B2 in all partitions. For T3, there is only one source block. Include B3 in all partitions. Partitions in Partitioned Compaction Group:\nPartition ID: 0 Number of Partitions: 2 Blocks: B1, B2, B3 Partition ID: 1 Number of Partitions: 2 Blocks: B1, B2, B3 Scenario: All source blocks are with compaction level \u003e 1 and some of them were generated by compactor with partitioning compaction Blocks generated by compactor without partitioning compaction would be included in all partitions. Blocks generated with partitioning compaction would be partitioned based on multiplier.\nTime ranges: T1, T2, T3 Source blocks: T1: B1 (unpartitioned) T2: B2, B3 T3: B4, B5, B6, B7 Total indices size of all source blocks: 100G Number of Partitions = (100G / 64G = 1.5625) =\u003e round up to next 2^x = 2\nPartitioning:\nFor T1, there is only one source block. Include B1 in all partitions. For T2, B2 (index 0 in the time range) can be grouped with other blocks having N % 2 == 0. B3 (index 1 in the time range) can be grouped with other blocks having N % 2 == 1. For T3, B4 (index 0 in the time range) can be grouped with other blocks having N % 2 == 0. B5 (index 1 in the time range) can be grouped with other blocks having N % 2 == 1. B6 (index 2 in the time range) can be grouped with other blocks having N % 2 == 0. B7 (index 3 in the time range) can be grouped with other blocks having N % 2 == 1. Partitions in Partitioned Compaction Group:\nPartition ID: 0 Number of Partitions: 2 Blocks: B1, B2, B4, B6 Partition ID: 1 Number of Partitions: 2 Blocks: B1, B3, B5, B7 ","categories":"","description":"","excerpt":" Author: @roystchiang Reviewers: Date: August 2022 Status: Proposed …","ref":"/docs/proposals/timeseries-partitioning-in-compactor/","tags":"","title":"Timeseries Partitioning in Compactor"},{"body":"Cortex consists of multiple horizontally scalable microservices. Each microservice uses the most appropriate technique for horizontal scaling; most are stateless and can handle requests for any users while some (namely the ingesters) are semi-stateful and depend on consistent hashing. This document provides a basic overview of Cortex’s architecture.\nThe following diagram does not include all the Cortex services, but does represent a typical deployment topology.\nThe role of Prometheus Prometheus instances scrape samples from various targets and then push them to Cortex (using Prometheus’ remote write API). That remote write API emits batched Snappy-compressed Protocol Buffer messages inside the body of an HTTP PUT request.\nCortex requires that each HTTP request bear a header specifying a tenant ID for the request. Request authentication and authorization are handled by an external reverse proxy.\nIncoming samples (writes from Prometheus) are handled by the distributor while incoming reads (PromQL queries) are handled by the querier or optionally by the query frontend.\nBlocks storage The blocks storage is based on Prometheus TSDB: it stores each tenant’s time series into their own TSDB which write out their series to a on-disk Block (defaults to 2h block range periods). Each Block is composed by a few files storing the chunks and the block index.\nThe TSDB chunk files contain the samples for multiple series. The series inside the Chunks are then indexed by a per-block index, which indexes metric names and labels to time series in the chunk files.\nThe blocks storage doesn’t require a dedicated storage backend for the index. The only requirement is an object store for the Block files, which can be:\nAmazon S3 Google Cloud Storage Microsoft Azure Storage OpenStack Swift Local Filesystem (single node only) For more information, please check out the Blocks storage documentation.\nServices Cortex has a service-based architecture, in which the overall system is split up into a variety of components that perform a specific task. These components run separately and in parallel. Cortex can alternatively run in a single process mode, where all components are executed within a single process. The single process mode is particularly handy for local testing and development.\nThe Cortex services are:\nDistributor Ingester Querier Compactor Store gateway Alertmanager (optional) Configs API (optional) Overrides exporter (optional) Query frontend (optional) Query scheduler (optional) Ruler (optional) Distributor The distributor service is responsible for handling incoming samples from Prometheus. It’s the first stop in the write path for series samples. Once the distributor receives samples from Prometheus, each sample is validated for correctness and to ensure that it is within the configured tenant limits, falling back to default ones in case limits have not been overridden for the specific tenant. Valid samples are then split into batches and sent to multiple ingesters in parallel.\nThe validation done by the distributor includes:\nThe metric labels name are formally correct The configured max number of labels per metric is respected The configured max length of a label name and value is respected The timestamp is not older/newer than the configured min/max time range Distributors are stateless and can be scaled up and down as needed.\nHigh Availability Tracker The distributor features a High Availability (HA) Tracker. When enabled, the distributor deduplicates incoming samples from redundant Prometheus servers. This allows you to have multiple HA replicas of the same Prometheus servers, writing the same series to Cortex and then deduplicate these series in the Cortex distributor.\nThe HA Tracker deduplicates incoming samples based on a cluster and replica label. The cluster label uniquely identifies the cluster of redundant Prometheus servers for a given tenant, while the replica label uniquely identifies the replica within the Prometheus cluster. Incoming samples are considered duplicated (and thus dropped) if received by any replica which is not the current primary within a cluster.\nThe HA Tracker requires a key-value (KV) store to coordinate which replica is currently elected. The distributor will only accept samples from the current leader. Samples with one or no labels (of the replica and cluster) are accepted by default and never deduplicated.\nThe supported KV stores for the HA tracker are:\nConsul Etcd Note: Memberlist is not supported. Memberlist-based KV store propagates updates using gossip, which is very slow for HA purposes: result is that different distributors may see different Prometheus server as elected HA replica, which is definitely not desirable.\nFor more information, please refer to config for sending HA pairs data to Cortex in the documentation.\nHashing Distributors use consistent hashing, in conjunction with a configurable replication factor, to determine which ingester instance(s) should receive a given series.\nCortex supports two hashing strategies:\nHash the metric name and tenant ID (default) Hash the metric name, labels and tenant ID (enabled with -distributor.shard-by-all-labels=true) The trade-off associated with the latter is that writes are more balanced across ingesters but each query needs to talk to all ingesters since a metric could be spread across multiple ingesters given different label sets.\nThe hash ring A hash ring (stored in a key-value store) is used to achieve consistent hashing for the series sharding and replication across the ingesters. All ingesters register themselves into the hash ring with a set of tokens they own; each token is a random unsigned 32-bit number. Each incoming series is hashed in the distributor and then pushed to the ingester owning the tokens range for the series hash number plus N-1 subsequent ingesters in the ring, where N is the replication factor.\nTo do the hash lookup, distributors find the smallest appropriate token whose value is larger than the hash of the series. When the replication factor is larger than 1, the next subsequent tokens (clockwise in the ring) that belong to different ingesters will also be included in the result.\nThe effect of this hash set up is that each token that an ingester owns is responsible for a range of hashes. If there are three tokens with values 0, 25, and 50, then a hash of 3 would be given to the ingester that owns the token 25; the ingester owning token 25 is responsible for the hash range of 1-25.\nThe supported KV stores for the hash ring are:\nConsul Etcd Gossip memberlist Quorum consistency Since all distributors share access to the same hash ring, write requests can be sent to any distributor and you can setup a stateless load balancer in front of it.\nTo ensure consistent query results, Cortex uses Dynamo-style quorum consistency on reads and writes. This means that the distributor will wait for a positive response of at least one half plus one of the ingesters to send the sample to before successfully responding to the Prometheus write request.\nLoad balancing across distributors We recommend randomly load balancing write requests across distributor instances. For example, if you’re running Cortex in a Kubernetes cluster, you could run the distributors as a Kubernetes Service.\nIngester The ingester service is responsible for writing incoming series to a long-term storage backend on the write path and returning in-memory series samples for queries on the read path.\nIncoming series are not immediately written to the storage but kept in memory and periodically flushed to the storage (by default, 2 hours). For this reason, the queriers may need to fetch samples both from ingesters and long-term storage while executing a query on the read path.\nIngesters contain a lifecycler which manages the lifecycle of an ingester and stores the ingester state in the hash ring. Each ingester could be in one of the following states:\nPENDING\nThe ingester has just started. While in this state, the ingester doesn’t receive neither write and read requests. JOINING\nThe ingester is starting up and joining the ring. While in this state the ingester doesn’t receive neither write and read requests. The ingester will join the ring using tokens loaded from disk (if -ingester.tokens-file-path is configured) or generate a set of new random ones. Finally, the ingester optionally observes the ring for tokens conflicts and then, once any conflict is resolved, will move to ACTIVE state. ACTIVE\nThe ingester is up and running. While in this state the ingester can receive both write and read requests. LEAVING\nThe ingester is shutting down and leaving the ring. While in this state the ingester doesn’t receive write requests, while it could receive read requests. UNHEALTHY\nThe ingester has failed to heartbeat to the ring’s KV Store. While in this state, distributors skip the ingester while building the replication set for incoming series and the ingester does not receive write or read requests. Ingesters are semi-stateful.\nIngesters failure and data loss If an ingester process crashes or exits abruptly, all the in-memory series that have not yet been flushed to the long-term storage will be lost. There are two main ways to mitigate this failure mode:\nReplication Write-ahead log (WAL) The replication is used to hold multiple (typically 3) replicas of each time series in the ingesters. If the Cortex cluster loses an ingester, the in-memory series held by the lost ingester are also replicated to at least another ingester. In the event of a single ingester failure, no time series samples will be lost. However, in the event of multiple ingester failures, time series may be potentially lost if the failures affect all the ingesters holding the replicas of a specific time series.\nThe write-ahead log (WAL) is used to write to a persistent disk all incoming series samples until they’re flushed to the long-term storage. In the event of an ingester failure, a subsequent process restart will replay the WAL and recover the in-memory series samples.\nContrary to the sole replication and given the persistent disk data is not lost, in the event of multiple ingesters failure each ingester will recover the in-memory series samples from WAL upon subsequent restart. The replication is still recommended in order to ensure no temporary failures on the read path in the event of a single ingester failure.\nIngesters write de-amplification Ingesters store recently received samples in-memory in order to perform write de-amplification. If the ingesters would immediately write received samples to the long-term storage, the system would be very difficult to scale due to the very high pressure on the storage. For this reason, the ingesters batch and compress samples in-memory and periodically flush them out to the storage.\nWrite de-amplification is the main source of Cortex’s low total cost of ownership (TCO).\nQuerier The querier service handles queries using the PromQL query language.\nQueriers fetch series samples both from the ingesters and long-term storage. The ingesters hold the in-memory series that have not yet been flushed to the long-term storage. Because of the replication factor, it is possible that the querier may receive duplicated samples. To address this, the querier internally deduplicates samples with the exact same timestamp for a given time series.\nQueriers are stateless and can be scaled up and down as needed.\nCompactor The compactor is a service which is responsible to:\nCompact multiple blocks of a given tenant into a single optimized larger block. This helps to reduce storage costs (deduplication, index size reduction), and increase query speed (querying fewer blocks is faster). Keep the per-tenant bucket index updated. The bucket index is used by queriers, store-gateways and rulers to discover new blocks in the storage. For more information, see the compactor documentation.\nThe compactor is stateless.\nStore gateway The store gateway is the Cortex service responsible for querying series from blocks. It needs to have an almost up-to-date view of the storage bucket. To discover blocks belonging to their shard, the store gateway can keep the bucket view updated in two different ways:\nPeriodically scanning the bucket (default) Periodically downloading the bucket index For more information, see the store gateway documentation.\nThe store gateway is semi-stateful.\nQuery frontend The query frontend is an optional service providing the querier’s API endpoints and can be used to accelerate the read path. When the query frontend is in place, incoming query requests should be directed to the query frontend instead of the queriers. The querier service will be still required within the cluster, in order to execute the actual queries.\nThe query frontend internally performs some query adjustments and holds queries in an internal queue. In this setup, queriers act as workers which pull jobs from the queue, execute them, and return them to the query-frontend for aggregation. Queriers need to be configured with the query frontend address (via the -querier.frontend-address CLI flag) in order to allow them to connect to the query frontends.\nQuery frontends are stateless. However, due to how the internal queue works, it’s recommended to run a few query frontend replicas to reap the benefit of fair scheduling. Two replicas should suffice in most cases.\nFlow of the query in the system when using query-frontend:\nQuery is received by query frontend, which can optionally split it or serve from the cache. Query frontend stores the query into in-memory queue, where it waits for some querier to pick it up. Querier picks up the query, and executes it. Querier sends result back to query-frontend, which then forwards it to the client. Query frontend can also be used with any Prometheus-API compatible service. In this mode Cortex can be used as an query accelerator with it’s caching and splitting features on other prometheus query engines like Thanos Querier or your own Prometheus server. Query frontend needs to be configured with downstream url address(via the -frontend.downstream-url CLI flag), which is the endpoint of the prometheus server intended to be connected with Cortex.\nQueueing The query frontend queuing mechanism is used to:\nEnsure that large queries, that could cause an out-of-memory (OOM) error in the querier, will be retried on failure. This allows administrators to under-provision memory for queries, or optimistically run more small queries in parallel, which helps to reduce the total cost of ownership (TCO). Prevent multiple large requests from being convoyed on a single querier by distributing them across all queriers using a first-in/first-out queue (FIFO). Prevent a single tenant from denial-of-service-ing (DOSing) other tenants by fairly scheduling queries between tenants. Splitting The query frontend splits multi-day queries into multiple single-day queries, executing these queries in parallel on downstream queriers and stitching the results back together again. This prevents large (multi-day) queries from causing out of memory issues in a single querier and helps to execute them faster.\nCaching The query frontend supports caching query results and reuses them on subsequent queries. If the cached results are incomplete, the query frontend calculates the required subqueries and executes them in parallel on downstream queriers. The query frontend can optionally align queries with their step parameter to improve the cacheability of the query results. The result cache is compatible with any cortex caching backend (currently memcached, redis, and an in-memory cache).\nQuery Scheduler Query Scheduler is an optional service that moves the internal queue from query frontend into separate component. This enables independent scaling of query frontends and number of queues (query scheduler).\nIn order to use query scheduler, both query frontend and queriers must be configured with query scheduler address (using -frontend.scheduler-address and -querier.scheduler-address options respectively).\nFlow of the query in the system changes when using query scheduler:\nQuery is received by query frontend, which can optionally split it or serve from the cache. Query frontend forwards the query to random query scheduler process. Query scheduler stores the query into in-memory queue, where it waits for some querier to pick it up. Querier picks up the query, and executes it. Querier sends result back to query-frontend, which then forwards it to the client. Query schedulers are stateless. It is recommended to run two replicas to make sure queries can still be serviced while one replica is restarting.\nRuler The ruler is an optional service executing PromQL queries for recording rules and alerts. The ruler requires a database storing the recording rules and alerts for each tenant.\nRuler is semi-stateful and can be scaled horizontally. Running rules internally have state, as well as the ring the rulers initiate. However, if the rulers all fail and restart, Prometheus alert rules have a feature where an alert is restored and returned to a firing state if it would have been active in its for period. However, there would be gaps in the series generated by the recording rules.\nAlertmanager The alertmanager is an optional service responsible for accepting alert notifications from the ruler, deduplicating and grouping them, and routing them to the correct notification channel, such as email, PagerDuty or OpsGenie.\nThe Cortex alertmanager is built on top of the Prometheus Alertmanager, adding multi-tenancy support. Like the ruler, the alertmanager requires a database storing the per-tenant configuration.\nAlertmanager is semi-stateful. The Alertmanager persists information about silences and active alerts to its disk. If all of the alertmanager nodes failed simultaneously there would be a loss of data.\nConfigs API The configs API is an optional service managing the configuration of Rulers and Alertmanagers. It provides APIs to get/set/update the ruler and alertmanager configurations and store them into backend. Current supported backend are PostgreSQL and in-memory.\nConfigs API is stateless.\n","categories":"","description":"","excerpt":"Cortex consists of multiple horizontally scalable microservices. Each …","ref":"/docs/architecture/","tags":"","title":"Cortex Architecture"},{"body":"General Notes Cortex has evolved over several years, and the command-line options sometimes reflect this heritage. In some cases the default value for options is not the recommended value, and in some cases names do not reflect the true meaning. We do intend to clean this up, but it requires a lot of care to avoid breaking existing installations. In the meantime we regret the inconvenience.\nDuration arguments should be specified with a unit like 5s or 3h. Valid time units are “ms”, “s”, “m”, “h”.\nQuerier -querier.max-concurrent\nThe maximum number of top-level PromQL queries that will execute at the same time, per querier process. If using the query frontend, this should be set to at least (-querier.worker-parallelism * number of query frontend replicas). Otherwise queries may queue in the queriers and not the frontend, which will affect QoS. Alternatively, consider using -querier.worker-match-max-concurrent to force worker parallelism to match -querier.max-concurrent.\n-querier.timeout\nThe timeout for a top-level PromQL query.\n-querier.max-samples\nMaximum number of samples a single query can load into memory, to avoid blowing up on enormous queries.\nThe next three options only apply when the querier is used together with the Query Frontend or Query Scheduler:\n-querier.frontend-address\nAddress of query frontend service, used by workers to find the frontend which will give them queries to execute.\n-querier.scheduler-address\nAddress of query scheduler service, used by workers to find the scheduler which will give them queries to execute. If set, -querier.frontend-address is ignored, and querier will use query scheduler.\n-querier.dns-lookup-period\nHow often the workers will query DNS to re-check where the query frontend or query scheduler is.\n-querier.worker-parallelism\nNumber of simultaneous queries to process, per query frontend or scheduler. See note on -querier.max-concurrent\n-querier.worker-match-max-concurrent\nForce worker concurrency to match the -querier.max-concurrent option. Overrides -querier.worker-parallelism. See note on -querier.max-concurrent\nQuerier and Ruler -promql.lookback-delta\nTime since the last sample after which a time series is considered stale and ignored by expression evaluations.\nQuery Frontend -querier.align-querier-with-step\nIf set to true, will cause the query frontend to mutate incoming queries and align their start and end parameters to the step parameter of the query. This improves the cacheability of the query results.\n-querier.split-queries-by-day\nIf set to true, will cause the query frontend to split multi-day queries into multiple single-day queries and execute them in parallel.\n-querier.cache-results\nIf set to true, will cause the querier to cache query results. The cache will be used to answer future, overlapping queries. The query frontend calculates extra queries required to fill gaps in the cache.\n-frontend.forward-headers-list\nRequest headers forwarded by query frontend to downstream queriers. Multiple headers may be specified. Defaults to empty.\n-frontend.max-cache-freshness\nWhen caching query results, it is desirable to prevent the caching of very recent results that might still be in flux. Use this parameter to configure the age of results that should be excluded.\n-frontend.memcached.{hostname, service, timeout}\nUse these flags to specify the location and timeout of the memcached cluster used to cache query results.\n-frontend.redis.{endpoint, timeout}\nUse these flags to specify the location and timeout of the Redis service used to cache query results.\nDistributor -distributor.shard-by-all-labels\nIn the original Cortex design, samples were sharded amongst distributors by the combination of (userid, metric name). Sharding by metric name was designed to reduce the number of ingesters you need to hit on the read path; the downside was that you could hotspot the write path.\nIn hindsight, this seems like the wrong choice: we do many orders of magnitude more writes than reads, and ingester reads are in-memory and cheap. It seems the right thing to do is to use all the labels to shard, improving load balancing and support for very high cardinality metrics.\nSet this flag to true for the new behaviour.\nImportant to note is that when setting this flag to true, it has to be set on both the distributor and the querier (called -distributor.shard-by-all-labels on Querier as well). If the flag is only set on the distributor and not on the querier, you will get incomplete query results because not all ingesters are queried.\nUpgrade notes: As this flag also makes all queries always read from all ingesters, the upgrade path is pretty trivial; just enable the flag. When you do enable it, you’ll see a spike in the number of active series as the writes are “reshuffled” amongst the ingesters, but over the next stale period all the old series will be flushed, and you should end up with much better load balancing. With this flag enabled in the queriers, reads will always catch all the data from all ingesters.\nWarning: disabling this flag can lead to a much less balanced distribution of load among the ingesters.\n-distributor.extra-query-delay This is used by a component with an embedded distributor (Querier and Ruler) to control how long to wait until sending more than the minimum amount of queries needed for a successful response.\ndistributor.ha-tracker.enable-for-all-users Flag to enable, for all users, handling of samples with external labels identifying replicas in an HA Prometheus setup. This defaults to false, and is technically defined in the Distributor limits.\ndistributor.ha-tracker.enable Enable the distributors HA tracker so that it can accept samples from Prometheus HA replicas gracefully (requires labels). Global (for distributors), this ensures that the necessary internal data structures for the HA handling are created. The option enable-for-all-users is still needed to enable ingestion of HA samples for all users.\ndistributor.drop-label This flag can be used to specify label names that to drop during sample ingestion within the distributor and can be repeated in order to drop multiple labels.\nRing/HA Tracker Store The KVStore client is used by both the Ring and HA Tracker (HA Tracker doesn’t support memberlist as KV store).\n{ring,distributor.ha-tracker}.prefix The prefix for the keys in the store. Should end with a /. For example with a prefix of foo/, the key bar would be stored under foo/bar. {ring,distributor.ha-tracker}.store Backend storage to use for the HA Tracker (consul, etcd, inmemory, multi). {ring,distributor.ring}.store Backend storage to use for the Ring (consul, etcd, inmemory, memberlist, multi). Consul By default these flags are used to configure Consul used for the ring. To configure Consul for the HA tracker, prefix these flags with distributor.ha-tracker.\nconsul.hostname Hostname and port of Consul. consul.acl-token ACL token used to interact with Consul. consul.client-timeout HTTP timeout when talking to Consul. consul.consistent-reads Enable consistent reads to Consul. etcd By default these flags are used to configure etcd used for the ring. To configure etcd for the HA tracker, prefix these flags with distributor.ha-tracker.\netcd.endpoints The etcd endpoints to connect to. etcd.dial-timeout The timeout for the etcd connection. etcd.max-retries The maximum number of retries to do for failed ops. etcd.tls-enabled Enable TLS. etcd.tls-cert-path The TLS certificate file path. etcd.tls-key-path The TLS private key file path. etcd.tls-ca-path The trusted CA file path. etcd.tls-insecure-skip-verify Skip validating server certificate. etcd.ping-without-stream-allowd' Enable/Disable PermitWithoutStream parameter memberlist Warning: memberlist KV works only for the hash ring, not for the HA Tracker, because propagation of changes is too slow for HA Tracker purposes.\nWhen using memberlist-based KV store, each node maintains its own copy of the hash ring. Updates generated locally, and received from other nodes are merged together to form the current state of the ring on the node. Updates are also propagated to other nodes. All nodes run the following two loops:\nEvery “gossip interval”, pick random “gossip nodes” number of nodes, and send recent ring updates to them. Every “push/pull sync interval”, choose random single node, and exchange full ring information with it (push/pull sync). After this operation, rings on both nodes are the same. When a node receives a ring update, node will merge it into its own ring state, and if that resulted in a change, node will add that update to the list of gossiped updates. Such update will be gossiped R * log(N+1) times by this node (R = retransmit multiplication factor, N = number of gossiping nodes in the cluster).\nIf you find the propagation to be too slow, there are some tuning possibilities (default values are memberlist settings for LAN networks):\nDecrease gossip interval (default: 200ms) Increase gossip nodes (default 3) Decrease push/pull sync interval (default 30s) Increase retransmit multiplication factor (default 4) To find propagation delay, you can use cortex_ring_oldest_member_timestamp{state=\"ACTIVE\"} metric.\nFlags for configuring KV store based on memberlist library:\nmemberlist.nodename Name of the node in memberlist cluster. Defaults to hostname. memberlist.randomize-node-name This flag adds extra random suffix to the node name used by memberlist. Defaults to true. Using random suffix helps to prevent issues when running multiple memberlist nodes on the same machine, or when node names are reused (eg. in stateful sets). memberlist.retransmit-factor Multiplication factor used when sending out messages (factor * log(N+1)). If not set, default value is used. memberlist.join Other cluster members to join. Can be specified multiple times. memberlist.min-join-backoff, memberlist.max-join-backoff, memberlist.max-join-retries These flags control backoff settings when joining the cluster. memberlist.abort-if-join-fails If this node fails to join memberlist cluster, abort. memberlist.rejoin-interval How often to try to rejoin the memberlist cluster. Defaults to 0, no rejoining. Occasional rejoin may be useful in some configurations, and is otherwise harmless. memberlist.left-ingesters-timeout How long to keep LEFT ingesters in the ring. Note: this is only used for gossiping, LEFT ingesters are otherwise invisible. memberlist.leave-timeout Timeout for leaving memberlist cluster. memberlist.gossip-interval How often to gossip with other cluster members. Uses memberlist LAN defaults if 0. memberlist.gossip-nodes How many nodes to gossip with in each gossip interval. Uses memberlist LAN defaults if 0. memberlist.pullpush-interval How often to use pull/push sync. Uses memberlist LAN defaults if 0. memberlist.bind-addr IP address to listen on for gossip messages. Multiple addresses may be specified. Defaults to 0.0.0.0. memberlist.bind-port Port to listen on for gossip messages. Defaults to 7946. memberlist.packet-dial-timeout Timeout used when connecting to other nodes to send packet. memberlist.packet-write-timeout Timeout for writing ‘packet’ data. memberlist.transport-debug Log debug transport messages. Note: global log.level must be at debug level as well. memberlist.gossip-to-dead-nodes-time How long to keep gossiping to the nodes that seem to be dead. After this time, dead node is removed from list of nodes. If “dead” node appears again, it will simply join the cluster again, if its name is not reused by other node in the meantime. If the name has been reused, such a reanimated node will be ignored by other members. memberlist.dead-node-reclaim-time How soon can dead’s node name be reused by a new node (using different IP). Disabled by default, name reclaim is not allowed until gossip-to-dead-nodes-time expires. This can be useful to set to low numbers when reusing node names, eg. in stateful sets. If memberlist library detects that new node is trying to reuse the name of previous node, it will log message like this: Conflicting address for ingester-6. Mine: 10.44.12.251:7946 Theirs: 10.44.12.54:7946 Old state: 2. Node states are: “alive” = 0, “suspect” = 1 (doesn’t respond, will be marked as dead if it doesn’t respond), “dead” = 2. Multi KV This is a special key-value implementation that uses two different KV stores (eg. consul, etcd or memberlist). One of them is always marked as primary, and all reads and writes go to primary store. Other one, secondary, is only used for writes. The idea is that operator can use multi KV store to migrate from primary to secondary store in runtime.\nFor example, migration from Consul to Etcd would look like this:\nSet ring.store to use multi store. Set -multi.primary=consul and -multi.secondary=etcd. All consul and etcd settings must still be specified. Start all Cortex microservices. They will still use Consul as primary KV, but they will also write share ring via etcd. Operator can now use “runtime config” mechanism to switch primary store to etcd. After all Cortex microservices have picked up new primary store, and everything looks correct, operator can now shut down Consul, and modify Cortex configuration to use -ring.store=etcd only. At this point, Consul can be shut down. Multi KV has following parameters:\nmulti.primary - name of primary KV store. Same values as in ring.store are supported, except multi. multi.secondary - name of secondary KV store. multi.mirror-enabled - enable mirroring of values to secondary store, defaults to true multi.mirror-timeout - wait max this time to write to secondary store to finish. Default to 2 seconds. Errors writing to secondary store are not reported to caller, but are logged and also reported via cortex_multikv_mirror_write_errors_total metric. Multi KV also reacts on changes done via runtime configuration. It uses this section:\nmulti_kv_config: mirror_enabled: false primary: memberlist Note that runtime configuration values take precedence over command line options.\nHA Tracker HA tracking has two of its own flags:\ndistributor.ha-tracker.cluster Prometheus label to look for in samples to identify a Prometheus HA cluster. (default “cluster”) distributor.ha-tracker.replica Prometheus label to look for in samples to identify a Prometheus HA replica. (default “__replica__”) It’s reasonable to assume people probably already have a cluster label, or something similar. If not, they should add one along with __replica__ via external labels in their Prometheus config. If you stick to these default values your Prometheus config could look like this (POD_NAME is an environment variable which must be set by you):\nglobal: external_labels: cluster: clustername __replica__: $POD_NAME HA Tracking looks for the two labels (which can be overwritten per user)\nIt also talks to a KVStore and has it’s own copies of the same flags used by the Distributor to connect to for the ring.\ndistributor.ha-tracker.failover-timeout If we don’t receive any samples from the accepted replica for a cluster in this amount of time we will failover to the next replica we receive a sample from. This value must be greater than the update timeout (default 30s) distributor.ha-tracker.store Backend storage to use for the ring (consul, etcd, inmemory, multi). Inmemory only works if there is a single distributor and ingester running in the same process (for testing purposes). (default “consul”) distributor.ha-tracker.update-timeout Update the timestamp in the KV store for a given cluster/replica only after this amount of time has passed since the current stored timestamp. (default 15s) Ingester -ingester.join-after\nHow long to wait in PENDING state during the hand-over process (supported only by the chunks storage). (default 0s)\n-ingester-client.expected-timeseries\nWhen push requests arrive, pre-allocate this many slots to decode them. Tune this setting to reduce memory allocations and garbage. This should match the max_samples_per_send in your queue_config for Prometheus.\n-ingester-client.expected-samples-per-series\nWhen push requests arrive, pre-allocate this many slots to decode them. Tune this setting to reduce memory allocations and garbage. Under normal conditions, Prometheus scrapes should arrive with one sample per series.\n-ingester-client.expected-labels\nWhen push requests arrive, pre-allocate this many slots to decode them. Tune this setting to reduce memory allocations and garbage. The optimum value will depend on how many labels are sent with your timeseries samples.\nRuntime Configuration file Cortex has a concept of “runtime config” file, which is simply a file that is reloaded while Cortex is running. It is used by some Cortex components to allow operator to change some aspects of Cortex configuration without restarting it. File is specified by using -runtime-config.file=\u003cfilename\u003e flag and reload period (which defaults to 10 seconds) can be changed by -runtime-config.reload-period=\u003cduration\u003e flag. Previously this mechanism was only used by limits overrides, and flags were called -limits.per-user-override-config=\u003cfilename\u003e and -limits.per-user-override-period=10s respectively. These are still used, if -runtime-config.file=\u003cfilename\u003e is not specified.\nAt the moment runtime configuration may contain per-user limits, multi KV store, and ingester instance limits.\nExample runtime configuration file:\noverrides: tenant1: ingestion_rate: 10000 max_series_per_metric: 100000 max_series_per_query: 100000 tenant2: max_samples_per_query: 1000000 max_series_per_metric: 100000 max_series_per_query: 100000 multi_kv_config: mirror_enabled: false primary: memberlist ingester_limits: max_ingestion_rate: 42000 max_inflight_push_requests: 10000 When running Cortex on Kubernetes, store this file in a config map and mount it in each services’ containers. When changing the values there is no need to restart the services, unless otherwise specified.\nThe /runtime_config endpoint returns the whole runtime configuration, including the overrides. In case you want to get only the non-default values of the configuration you can pass the mode parameter with the diff value.\nIngester, Distributor \u0026 Querier limits. Cortex implements various limits on the requests it can process, in order to prevent a single tenant overwhelming the cluster. There are various default global limits which apply to all tenants which can be set on the command line. These limits can also be overridden on a per-tenant basis by using overrides field of runtime configuration file.\nThe overrides field is a map of tenant ID (same values as passed in the X-Scope-OrgID header) to the various limits. An example could look like:\noverrides: tenant1: ingestion_rate: 10000 max_series_per_metric: 100000 max_series_per_query: 100000 tenant2: max_samples_per_query: 1000000 max_series_per_metric: 100000 max_series_per_query: 100000 Valid per-tenant limits are (with their corresponding flags for default values):\ningestion_rate_strategy / -distributor.ingestion-rate-limit-strategy\ningestion_rate / -distributor.ingestion-rate-limit\ningestion_burst_size / -distributor.ingestion-burst-size\nThe per-tenant rate limit (and burst size), in samples per second. It supports two strategies: local (default) and global.\nThe local strategy enforces the limit on a per distributor basis, actual effective rate limit will be N times higher, where N is the number of distributor replicas.\nThe global strategy enforces the limit globally, configuring a per-distributor local rate limiter as ingestion_rate / N, where N is the number of distributor replicas (it’s automatically adjusted if the number of replicas change). The ingestion_burst_size refers to the per-distributor local rate limiter (even in the case of the global strategy) and should be set at least to the maximum number of samples expected in a single push request. For this reason, the global strategy requires that push requests are evenly distributed across the pool of distributors; if you use a load balancer in front of the distributors you should be already covered, while if you have a custom setup (ie. an authentication gateway in front) make sure traffic is evenly balanced across distributors.\nThe global strategy requires the distributors to form their own ring, which is used to keep track of the current number of healthy distributor replicas. The ring is configured by distributor: { ring: {}} / -distributor.ring.*.\nmax_label_name_length / -validation.max-length-label-name\nmax_label_value_length / -validation.max-length-label-value\nmax_label_names_per_series / -validation.max-label-names-per-series\nAlso enforced by the distributor, limits on the on length of labels and their values, and the total number of labels allowed per series.\nreject_old_samples / -validation.reject-old-samples\nreject_old_samples_max_age / -validation.reject-old-samples.max-age\ncreation_grace_period / -validation.create-grace-period\nAlso enforce by the distributor, limits on how far in the past (and future) timestamps that we accept can be.\nmax_series_per_user / -ingester.max-series-per-user\nmax_series_per_metric / -ingester.max-series-per-metric\nEnforced by the ingesters; limits the number of active series a user (or a given metric) can have. When running with -distributor.shard-by-all-labels=false (the default), this limit will enforce the maximum number of series a metric can have ‘globally’, as all series for a single metric will be sent to the same replication set of ingesters. This is not the case when running with -distributor.shard-by-all-labels=true, so the actual limit will be N/RF times higher, where N is number of ingester replicas and RF is configured replication factor.\nmax_global_series_per_user / -ingester.max-global-series-per-user\nmax_global_series_per_metric / -ingester.max-global-series-per-metric\nLike max_series_per_user and max_series_per_metric, but the limit is enforced across the cluster. Each ingester is configured with a local limit based on the replication factor, the -distributor.shard-by-all-labels setting and the current number of healthy ingesters, and is kept updated whenever the number of ingesters change.\nRequires -distributor.replication-factor, -distributor.shard-by-all-labels, -distributor.sharding-strategy and -distributor.zone-awareness-enabled set for the ingesters too.\nmax_series_per_query / -ingester.max-series-per-query\nmax_samples_per_query / -ingester.max-samples-per-query\nLimits on the number of timeseries and samples returns by a single ingester during a query.\nmax_metadata_per_user / -ingester.max-metadata-per-user\nmax_metadata_per_metric / -ingester.max-metadata-per-metric Enforced by the ingesters; limits the number of active metadata a user (or a given metric) can have. When running with -distributor.shard-by-all-labels=false (the default), this limit will enforce the maximum number of metadata a metric can have ‘globally’, as all metadata for a single metric will be sent to the same replication set of ingesters. This is not the case when running with -distributor.shard-by-all-labels=true, so the actual limit will be N/RF times higher, where N is number of ingester replicas and RF is configured replication factor.\nmax_fetched_series_per_query / querier.max-fetched-series-per-query When running Cortex with blocks storage this limit is enforced in the queriers on unique series fetched from ingesters and store-gateways (long-term storage).\nmax_global_metadata_per_user / -ingester.max-global-metadata-per-user\nmax_global_metadata_per_metric / -ingester.max-global-metadata-per-metric\nLike max_metadata_per_user and max_metadata_per_metric, but the limit is enforced across the cluster. Each ingester is configured with a local limit based on the replication factor, the -distributor.shard-by-all-labels setting and the current number of healthy ingesters, and is kept updated whenever the number of ingesters change.\nRequires -distributor.replication-factor, -distributor.shard-by-all-labels, -distributor.sharding-strategy and -distributor.zone-awareness-enabled set for the ingesters too.\nIngester Instance Limits Cortex ingesters support limits that are applied per-instance, meaning they apply to each ingester process. These can be used to ensure individual ingesters are not overwhelmed regardless of any per-user limits. These limits can be set under the ingester.instance_limits block in the global configuration file, with command line flags, or under the ingester_limits field in the runtime configuration file.\nAn example as part of the runtime configuration file:\ningester_limits: max_ingestion_rate: 20000 max_series: 1500000 max_tenants: 1000 max_inflight_push_requests: 30000 Valid ingester instance limits are (with their corresponding flags):\nmax_ingestion_rate \\ --ingester.instance-limits.max-ingestion-rate\nLimit the ingestion rate in samples per second for an ingester. When this limit is reached, new requests will fail with an HTTP 500 error.\nmax_series \\ -ingester.instance-limits.max-series\nLimit the total number of series that an ingester keeps in memory, across all users. When this limit is reached, requests that create new series will fail with an HTTP 500 error.\nmax_tenants \\ -ingester.instance-limits.max-tenants\nLimit the maximum number of users an ingester will accept metrics for. When this limit is reached, requests from new users will fail with an HTTP 500 error.\nmax_inflight_push_requests \\ -ingester.instance-limits.max-inflight-push-requests\nLimit the maximum number of requests being handled by an ingester at once. This setting is critical for preventing ingesters from using an excessive amount of memory during high load or temporary slow downs. When this limit is reached, new requests will fail with an HTTP 500 error.\nStorage s3.force-path-style\nSet this to true to force the request to use path-style addressing (http://s3.amazonaws.com/BUCKET/KEY). By default, the S3 client will use virtual hosted bucket addressing when possible (http://BUCKET.s3.amazonaws.com/KEY).\nDNS Service Discovery Some clients in Cortex support service discovery via DNS to find addresses of backend servers to connect to (ie. caching servers). The clients supporting it are:\nBlocks storage’s memcached cache All caching memcached servers Memberlist KV store Supported discovery modes The DNS service discovery, inspired from Thanos DNS SD, supports different discovery modes. A discovery mode is selected adding a specific prefix to the address. The supported prefixes are:\ndns+\nThe domain name after the prefix is looked up as an A/AAAA query. For example: dns+memcached.local:11211 dnssrv+\nThe domain name after the prefix is looked up as a SRV query, and then each SRV record is resolved as an A/AAAA record. For example: dnssrv+_memcached._tcp.memcached.namespace.svc.cluster.local dnssrvnoa+\nThe domain name after the prefix is looked up as a SRV query, with no A/AAAA lookup made after that. For example: dnssrvnoa+_memcached._tcp.memcached.namespace.svc.cluster.local If no prefix is provided, the provided IP or hostname will be used straightaway without pre-resolving it.\nIf you are using a managed memcached service from Google Cloud, or AWS, use the auto-discovery flag instead of DNS discovery, then use the discovery/configuration endpoint as the domain name without any prefix.\nLogging of IP of reverse proxy If a reverse proxy is used in front of Cortex it might be difficult to troubleshoot errors. The following 3 settings can be used to log the IP address passed along by the reverse proxy in headers like X-Forwarded-For.\n-server.log_source_ips_enabled\nSet this to true to add logging of the IP when a Forwarded, X-Real-IP or X-Forwarded-For header is used. A field called sourceIPs will be added to error logs when data is pushed into Cortex.\n-server.log-source-ips-header\nHeader field storing the source IPs. It is only used if -server.log-source-ips-enabled is true and if -server.log-source-ips-regex is set. If not set the default Forwarded, X-Real-IP or X-Forwarded-For headers are searched.\n-server.log-source-ips-regex\nRegular expression for matching the source IPs. It should contain at least one capturing group the first of which will be returned. Only used if -server.log-source-ips-enabled is true and if -server.log-source-ips-header is set. If not set the default Forwarded, X-Real-IP or X-Forwarded-For headers are searched.\n","categories":"","description":"","excerpt":"General Notes Cortex has evolved over several years, and the …","ref":"/docs/configuration/arguments/","tags":"","title":"Cortex Arguments"},{"body":"Cortex adopts some design patterns and code conventions that we ask you to follow when contributing to the project. These conventions have been adopted based on the experience gained over the time and aim to enforce good coding practices and keep a consistent UX (ie. config).\nGo coding style Cortex follows the Go Code Review Comments styleguide and the Formatting and style section of Peter Bourgon’s Go: Best Practices for Production Environments.\nNo global variables Do not use global variables Prometheus metrics When registering a metric:\nDo not use a global variable for the metric Create and register the metric with promauto.With(reg) In any internal Cortex component, do not register the metric to the default prometheus registerer, but pick the registerer in input (ie. NewComponent(reg prometheus.Registerer)) Testing metrics:\nWhen writing using tests, test exported metrics using testutil.GatherAndCompare() Config file and CLI flags conventions Naming:\nConfig file options should be lowercase with words _ (underscore) separated (ie. memcached_client) CLI flags should be lowercase with words - (dash) separated (ie. memcached-client) When adding a new config option, look if a similar one already exists within the config and keep the same naming (ie. addresses for a list of network endpoints) Documentation:\nA CLI flag mentioned in the documentation or changelog should be always prefixed with a single - (dash) ","categories":"","description":"","excerpt":"Cortex adopts some design patterns and code conventions that we ask …","ref":"/docs/contributing/design-patterns-and-code-conventions/","tags":"","title":"Design patterns and Code conventions"},{"body":" The querier service handles queries using the PromQL query language. This document dives into the storage-specific details of the querier service. The general architecture documentation applies too.\nThe querier is stateless.\nHow it works The querier needs to have an almost up-to-date view over the entire storage bucket, in order to find the right blocks to lookup at query time. The querier can keep the bucket view updated in to two different ways:\nPeriodically scanning the bucket (default) Periodically downloading the bucket index Bucket index disabled (default) At startup, queriers iterate over the entire storage bucket to discover all tenants blocks and download the meta.json for each block. During this initial bucket scanning phase, a querier is not ready to handle incoming queries yet and its /ready readiness probe endpoint will fail.\nWhile running, queriers periodically iterate over the storage bucket to discover new tenants and recently uploaded blocks. Queriers do not download any content from blocks except a small meta.json file containing the block’s metadata (including the minimum and maximum timestamp of samples within the block).\nQueriers use the metadata to compute the list of blocks that need to be queried at query time and fetch matching series from the store-gateway instances holding the required blocks.\nBucket index enabled When bucket index is enabled, queriers lazily download the bucket index upon the first query received for a given tenant, cache it in memory and periodically keep it update. The bucket index contains the list of blocks and block deletion marks of a tenant, which is later used during the query execution to find the set of blocks that need to be queried for the given query.\nGiven the bucket index removes the need to scan the bucket, it brings few benefits:\nThe querier is expected to be ready shortly after startup. Lower volume of API calls to object storage. Anatomy of a query request When a querier receives a query range request, it contains the following parameters:\nquery: the PromQL query expression itself (e.g. rate(node_cpu_seconds_total[1m])) start: the start time end: the end time step: the query resolution (e.g. 30 to have 1 resulting data point every 30s) Given a query, the querier analyzes the start and end time range to compute a list of all known blocks containing at least 1 sample within this time range. Given the list of blocks, the querier then computes a list of store-gateway instances holding these blocks and sends a request to each matching store-gateway instance asking to fetch all the samples for the series matching the query within the start and end time range.\nThe request sent to each store-gateway contains the list of block IDs that are expected to be queried, and the response sent back by the store-gateway to the querier contains the list of block IDs that were actually queried. This list may be a subset of the requested blocks, for example due to recent blocks resharding event (ie. last few seconds). The querier runs a consistency check on responses received from the store-gateways to ensure all expected blocks have been queried; if not, the querier retries to fetch samples from missing blocks from different store-gateways (if the -store-gateway.sharding-ring.replication-factor is greater than 1) and if the consistency check fails after all retries, the query execution fails as well (correctness is always guaranteed).\nIf the query time range covers a period within -querier.query-ingesters-within duration, the querier also sends the request to all ingesters, in order to fetch samples that have not been uploaded to the long-term storage yet.\nOnce all samples have been fetched from both store-gateways and ingesters, the querier proceeds with running the PromQL engine to execute the query and send back the result to the client.\nHow queriers connect to store-gateway Queriers need to discover store-gateways in order to connect to them at query time. The service discovery mechanism used depends whether blocks sharding is enabled in the store-gateways.\nWhen blocks sharding is enabled, queriers need to access to the store-gateways hash ring and thus queriers need to be configured with the same -store-gateway.sharding-ring.* flags (or their respective YAML config options) store-gateways have been configured.\nWhen blocks sharding is disabled, queriers need the -querier.store-gateway-addresses CLI flag (or its respective YAML config option) being set to a comma separated list of store-gateway addresses in DNS Service Discovery format. Queriers will evenly balance the requests to query blocks across the resolved addresses.\nCaching The querier supports the following caches:\nMetadata cache Caching is optional, but highly recommended in a production environment. Please also check out the production tips for more information about configuring the cache.\nMetadata cache Store-gateway and querier can use memcached for caching bucket metadata:\nList of tenants List of blocks per tenant Block’s meta.json content Block’s deletion-mark.json existence and content Tenant’s bucket-index.json.gz content Using the metadata cache can significantly reduce the number of API calls to object storage and protects from linearly scale the number of these API calls with the number of querier and store-gateway instances (because the bucket is periodically scanned and synched by each querier and store-gateway).\nTo enable metadata cache, please set -blocks-storage.bucket-store.metadata-cache.backend. Only memcached backend is supported currently. Memcached client has additional configuration available via flags with -blocks-storage.bucket-store.metadata-cache.memcached.* prefix.\nAdditional options for configuring metadata cache have -blocks-storage.bucket-store.metadata-cache.* prefix. By configuring TTL to zero or negative value, caching of given item type is disabled.\nThe same memcached backend cluster should be shared between store-gateways and queriers.\nQuerier configuration This section described the querier configuration. For the general Cortex configuration and references to common config blocks, please refer to the configuration documentation.\nquerier_config The querier_config configures the Cortex querier.\nquerier: # The maximum number of concurrent queries. # CLI flag: -querier.max-concurrent [max_concurrent: \u003cint\u003e | default = 20] # The timeout for a query. # CLI flag: -querier.timeout [timeout: \u003cduration\u003e | default = 2m] # Deprecated (This feature will be always on after v1.18): Use streaming RPCs # for metadata APIs from ingester. # CLI flag: -querier.ingester-metadata-streaming [ingester_metadata_streaming: \u003cboolean\u003e | default = true] # Use LabelNames ingester RPCs with match params. # CLI flag: -querier.ingester-label-names-with-matchers [ingester_label_names_with_matchers: \u003cboolean\u003e | default = false] # Maximum number of samples a single query can load into memory. # CLI flag: -querier.max-samples [max_samples: \u003cint\u003e | default = 50000000] # Maximum lookback beyond which queries are not sent to ingester. 0 means all # queries are sent to ingester. # CLI flag: -querier.query-ingesters-within [query_ingesters_within: \u003cduration\u003e | default = 0s] # Enable returning samples stats per steps in query response. # CLI flag: -querier.per-step-stats-enabled [per_step_stats_enabled: \u003cboolean\u003e | default = false] # Use compression for metrics query API or instant and range query APIs. # Supports 'gzip' and '' (disable compression) # CLI flag: -querier.response-compression [response_compression: \u003cstring\u003e | default = \"gzip\"] # The time after which a metric should be queried from storage and not just # ingesters. 0 means all queries are sent to store. When running the blocks # storage, if this option is enabled, the time range of the query sent to the # store will be manipulated to ensure the query end is not more recent than # 'now - query-store-after'. # CLI flag: -querier.query-store-after [query_store_after: \u003cduration\u003e | default = 0s] # Maximum duration into the future you can query. 0 to disable. # CLI flag: -querier.max-query-into-future [max_query_into_future: \u003cduration\u003e | default = 10m] # The default evaluation interval or step size for subqueries. # CLI flag: -querier.default-evaluation-interval [default_evaluation_interval: \u003cduration\u003e | default = 1m] # Max number of steps allowed for every subquery expression in query. Number # of steps is calculated using subquery range / step. A value \u003e 0 enables it. # CLI flag: -querier.max-subquery-steps [max_subquery_steps: \u003cint\u003e | default = 0] # Active query tracker monitors active queries, and writes them to the file in # given directory. If Cortex discovers any queries in this log during startup, # it will log them to the log file. Setting to empty value disables active # query tracker, which also disables -querier.max-concurrent option. # CLI flag: -querier.active-query-tracker-dir [active_query_tracker_dir: \u003cstring\u003e | default = \"./active-query-tracker\"] # Time since the last sample after which a time series is considered stale and # ignored by expression evaluations. # CLI flag: -querier.lookback-delta [lookback_delta: \u003cduration\u003e | default = 5m] # Comma separated list of store-gateway addresses in DNS Service Discovery # format. This option should be set when using the blocks storage and the # store-gateway sharding is disabled (when enabled, the store-gateway # instances form a ring and addresses are picked from the ring). # CLI flag: -querier.store-gateway-addresses [store_gateway_addresses: \u003cstring\u003e | default = \"\"] store_gateway_client: # Enable TLS for gRPC client connecting to store-gateway. # CLI flag: -querier.store-gateway-client.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for authenticating # with the server. Also requires the key path to be configured. # CLI flag: -querier.store-gateway-client.tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the client # certificate to be configured. # CLI flag: -querier.store-gateway-client.tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate against. # If not set, the host's root CA certificates are used. # CLI flag: -querier.store-gateway-client.tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -querier.store-gateway-client.tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -querier.store-gateway-client.tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # Use compression when sending messages. Supported values are: 'gzip', # 'snappy' and '' (disable compression) # CLI flag: -querier.store-gateway-client.grpc-compression [grpc_compression: \u003cstring\u003e | default = \"\"] # EXPERIMENTAL: If enabled, gRPC clients perform health checks for each # target and fail the request if the target is marked as unhealthy. healthcheck_config: # The number of consecutive failed health checks required before # considering a target unhealthy. 0 means disabled. # CLI flag: -querier.store-gateway-client.healthcheck.unhealthy-threshold [unhealthy_threshold: \u003cint\u003e | default = 0] # The approximate amount of time between health checks of an individual # target. # CLI flag: -querier.store-gateway-client.healthcheck.interval [interval: \u003cduration\u003e | default = 5s] # The amount of time during which no response from a target means a failed # health check. # CLI flag: -querier.store-gateway-client.healthcheck.timeout [timeout: \u003cduration\u003e | default = 1s] # If enabled, store gateway query stats will be logged using `info` log level. # CLI flag: -querier.store-gateway-query-stats-enabled [store_gateway_query_stats: \u003cboolean\u003e | default = true] # The maximum number of times we attempt fetching missing blocks from # different store-gateways. If no more store-gateways are left (ie. due to # lower replication factor) than we'll end the retries earlier # CLI flag: -querier.store-gateway-consistency-check-max-attempts [store_gateway_consistency_check_max_attempts: \u003cint\u003e | default = 3] # When distributor's sharding strategy is shuffle-sharding and this setting is # \u003e 0, queriers fetch in-memory series from the minimum set of required # ingesters, selecting only ingesters which may have received series since # 'now - lookback period'. The lookback period should be greater or equal than # the configured 'query store after' and 'query ingesters within'. If this # setting is 0, queriers always query all ingesters (ingesters shuffle # sharding on read path is disabled). # CLI flag: -querier.shuffle-sharding-ingesters-lookback-period [shuffle_sharding_ingesters_lookback_period: \u003cduration\u003e | default = 0s] # Experimental. Use Thanos promql engine # https://github.com/thanos-io/promql-engine rather than the Prometheus promql # engine. # CLI flag: -querier.thanos-engine [thanos_engine: \u003cboolean\u003e | default = false] # If enabled, ignore max query length check at Querier select method. Users # can choose to ignore it since the validation can be done before Querier # evaluation like at Query Frontend or Ruler. # CLI flag: -querier.ignore-max-query-length [ignore_max_query_length: \u003cboolean\u003e | default = false] # [Experimental] If true, experimental promQL functions are enabled. # CLI flag: -querier.enable-promql-experimental-functions [enable_promql_experimental_functions: \u003cboolean\u003e | default = false] blocks_storage_config The blocks_storage_config configures the blocks storage.\nblocks_storage: # Backend storage to use. Supported backends are: s3, gcs, azure, swift, # filesystem. # CLI flag: -blocks-storage.backend [backend: \u003cstring\u003e | default = \"s3\"] s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an # S3-compatible service in hostname:port format. # CLI flag: -blocks-storage.s3.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # S3 region. If unset, the client will issue a S3 GetBucketLocation API call # to autodetect it. # CLI flag: -blocks-storage.s3.region [region: \u003cstring\u003e | default = \"\"] # S3 bucket name # CLI flag: -blocks-storage.s3.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # S3 secret access key # CLI flag: -blocks-storage.s3.secret-access-key [secret_access_key: \u003cstring\u003e | default = \"\"] # S3 access key ID # CLI flag: -blocks-storage.s3.access-key-id [access_key_id: \u003cstring\u003e | default = \"\"] # If enabled, use http:// for the S3 endpoint instead of https://. This # could be useful in local dev/test environments while using an # S3-compatible backend storage, like Minio. # CLI flag: -blocks-storage.s3.insecure [insecure: \u003cboolean\u003e | default = false] # The signature version to use for authenticating against S3. Supported # values are: v4, v2. # CLI flag: -blocks-storage.s3.signature-version [signature_version: \u003cstring\u003e | default = \"v4\"] # The s3 bucket lookup style. Supported values are: auto, virtual-hosted, # path. # CLI flag: -blocks-storage.s3.bucket-lookup-type [bucket_lookup_type: \u003cstring\u003e | default = \"auto\"] # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum # algorithm to verify the provided digest. If false, use CRC32C algorithm # instead. # CLI flag: -blocks-storage.s3.send-content-md5 [send_content_md5: \u003cboolean\u003e | default = true] # The list api version. Supported values are: v1, v2, and ''. # CLI flag: -blocks-storage.s3.list-objects-version [list_objects_version: \u003cstring\u003e | default = \"\"] # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: blocks-storage [sse: \u003cs3_sse_config\u003e] http: # The time an idle connection will remain idle before closing. # CLI flag: -blocks-storage.s3.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -blocks-storage.s3.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -blocks-storage.s3.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -blocks-storage.s3.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully # writing the request headers if the request has an Expect header. 0 to # send the request body immediately. # CLI flag: -blocks-storage.s3.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 # means no limit. # CLI flag: -blocks-storage.s3.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, # a built-in default value is used. # CLI flag: -blocks-storage.s3.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -blocks-storage.s3.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] gcs: # GCS bucket name # CLI flag: -blocks-storage.gcs.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # JSON representing either a Google Developers Console # client_credentials.json file or a Google Developers service account key # file. If empty, fallback to Google default logic. # CLI flag: -blocks-storage.gcs.service-account [service_account: \u003cstring\u003e | default = \"\"] azure: # Azure storage account name # CLI flag: -blocks-storage.azure.account-name [account_name: \u003cstring\u003e | default = \"\"] # Azure storage account key # CLI flag: -blocks-storage.azure.account-key [account_key: \u003cstring\u003e | default = \"\"] # The values of `account-name` and `endpoint-suffix` values will not be # ignored if `connection-string` is set. Use this method over `account-key` # if you need to authenticate via a SAS token or if you use the Azurite # emulator. # CLI flag: -blocks-storage.azure.connection-string [connection_string: \u003cstring\u003e | default = \"\"] # Azure storage container name # CLI flag: -blocks-storage.azure.container-name [container_name: \u003cstring\u003e | default = \"\"] # Azure storage endpoint suffix without schema. The account name will be # prefixed to this value to create the FQDN # CLI flag: -blocks-storage.azure.endpoint-suffix [endpoint_suffix: \u003cstring\u003e | default = \"\"] # Number of retries for recoverable errors # CLI flag: -blocks-storage.azure.max-retries [max_retries: \u003cint\u003e | default = 20] # Deprecated: Azure storage MSI resource. It will be set automatically by # Azure SDK. # CLI flag: -blocks-storage.azure.msi-resource [msi_resource: \u003cstring\u003e | default = \"\"] # Azure storage MSI resource managed identity client Id. If not supplied # default Azure credential will be used. Set it to empty if you need to # authenticate via Azure Workload Identity. # CLI flag: -blocks-storage.azure.user-assigned-id [user_assigned_id: \u003cstring\u003e | default = \"\"] http: # The time an idle connection will remain idle before closing. # CLI flag: -blocks-storage.azure.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -blocks-storage.azure.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -blocks-storage.azure.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -blocks-storage.azure.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully # writing the request headers if the request has an Expect header. 0 to # send the request body immediately. # CLI flag: -blocks-storage.azure.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 # means no limit. # CLI flag: -blocks-storage.azure.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, # a built-in default value is used. # CLI flag: -blocks-storage.azure.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -blocks-storage.azure.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] swift: # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -blocks-storage.swift.auth-version [auth_version: \u003cint\u003e | default = 0] # OpenStack Swift authentication URL # CLI flag: -blocks-storage.swift.auth-url [auth_url: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential ID. # CLI flag: -blocks-storage.swift.application-credential-id [application_credential_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential name. # CLI flag: -blocks-storage.swift.application-credential-name [application_credential_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential secret. # CLI flag: -blocks-storage.swift.application-credential-secret [application_credential_secret: \u003cstring\u003e | default = \"\"] # OpenStack Swift username. # CLI flag: -blocks-storage.swift.username [username: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -blocks-storage.swift.user-domain-name [user_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -blocks-storage.swift.user-domain-id [user_domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user ID. # CLI flag: -blocks-storage.swift.user-id [user_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift API key. # CLI flag: -blocks-storage.swift.password [password: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -blocks-storage.swift.domain-id [domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -blocks-storage.swift.domain-name [domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift project ID (v2,v3 auth only). # CLI flag: -blocks-storage.swift.project-id [project_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift project name (v2,v3 auth only). # CLI flag: -blocks-storage.swift.project-name [project_name: \u003cstring\u003e | default = \"\"] # ID of the OpenStack Swift project's domain (v3 auth only), only needed if # it differs the from user domain. # CLI flag: -blocks-storage.swift.project-domain-id [project_domain_id: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift project's domain (v3 auth only), only needed # if it differs from the user domain. # CLI flag: -blocks-storage.swift.project-domain-name [project_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift Region to use (v2,v3 auth only). # CLI flag: -blocks-storage.swift.region-name [region_name: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift container to put chunks in. # CLI flag: -blocks-storage.swift.container-name [container_name: \u003cstring\u003e | default = \"\"] # Max retries on requests error. # CLI flag: -blocks-storage.swift.max-retries [max_retries: \u003cint\u003e | default = 3] # Time after which a connection attempt is aborted. # CLI flag: -blocks-storage.swift.connect-timeout [connect_timeout: \u003cduration\u003e | default = 10s] # Time after which an idle request is aborted. The timeout watchdog is reset # each time some data is received, so the timeout triggers after X time no # data is received on a request. # CLI flag: -blocks-storage.swift.request-timeout [request_timeout: \u003cduration\u003e | default = 5s] filesystem: # Local filesystem storage directory. # CLI flag: -blocks-storage.filesystem.dir [dir: \u003cstring\u003e | default = \"\"] # This configures how the querier and store-gateway discover and synchronize # blocks stored in the bucket. bucket_store: # Directory to store synchronized TSDB index headers. # CLI flag: -blocks-storage.bucket-store.sync-dir [sync_dir: \u003cstring\u003e | default = \"tsdb-sync\"] # How frequently to scan the bucket, or to refresh the bucket index (if # enabled), in order to look for changes (new blocks shipped by ingesters # and blocks deleted by retention or compaction). # CLI flag: -blocks-storage.bucket-store.sync-interval [sync_interval: \u003cduration\u003e | default = 15m] # Max number of concurrent queries to execute against the long-term storage. # The limit is shared across all tenants. # CLI flag: -blocks-storage.bucket-store.max-concurrent [max_concurrent: \u003cint\u003e | default = 100] # Max number of inflight queries to execute against the long-term storage. # The limit is shared across all tenants. 0 to disable. # CLI flag: -blocks-storage.bucket-store.max-inflight-requests [max_inflight_requests: \u003cint\u003e | default = 0] # Maximum number of concurrent tenants syncing blocks. # CLI flag: -blocks-storage.bucket-store.tenant-sync-concurrency [tenant_sync_concurrency: \u003cint\u003e | default = 10] # Maximum number of concurrent blocks syncing per tenant. # CLI flag: -blocks-storage.bucket-store.block-sync-concurrency [block_sync_concurrency: \u003cint\u003e | default = 20] # Number of Go routines to use when syncing block meta files from object # storage per tenant. # CLI flag: -blocks-storage.bucket-store.meta-sync-concurrency [meta_sync_concurrency: \u003cint\u003e | default = 20] # Minimum age of a block before it's being read. Set it to safe value (e.g # 30m) if your object storage is eventually consistent. GCS and S3 are # (roughly) strongly consistent. # CLI flag: -blocks-storage.bucket-store.consistency-delay [consistency_delay: \u003cduration\u003e | default = 0s] index_cache: # The index cache backend type. Multiple cache backend can be provided as # a comma-separated ordered list to enable the implementation of a cache # hierarchy. Supported values: inmemory, memcached, redis. # CLI flag: -blocks-storage.bucket-store.index-cache.backend [backend: \u003cstring\u003e | default = \"inmemory\"] inmemory: # Maximum size in bytes of in-memory index cache used to speed up blocks # index lookups (shared between all tenants). # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes [max_size_bytes: \u003cint\u003e | default = 1073741824] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items [enabled_items: \u003clist of string\u003e | default = []] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV # query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup # made after that). # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. # If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should # run. If more keys are specified, internally keys are split into # multiple batches and fetched concurrently, honoring the max # concurrency. If set to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items [enabled_items: \u003clist of string\u003e | default = []] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.index-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching # is when data is stored in memory instead of fetching data each time. # See https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items [enabled_items: \u003clist of string\u003e | default = []] multilevel: # The maximum number of concurrent asynchronous operations can occur # when backfilling cache items. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of items to backfill per asynchronous operation. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-backfill-items [max_backfill_items: \u003cint\u003e | default = 10000] chunks_cache: # The chunks cache backend type. Single or Multiple cache backend can be # provided. Supported values in single cache: memcached, redis, inmemory, # and '' (disable). Supported values in multi level cache: a # comma-separated list of (inmemory, memcached, redis) # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend [backend: \u003cstring\u003e | default = \"\"] inmemory: # Maximum size in bytes of in-memory chunk cache used to speed up chunk # lookups (shared between all tenants). # CLI flag: -blocks-storage.bucket-store.chunks-cache.inmemory.max-size-bytes [max_size_bytes: \u003cint\u003e | default = 1073741824] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV # query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup # made after that). # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. # If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should # run. If more keys are specified, internally keys are split into # multiple batches and fetched concurrently, honoring the max # concurrency. If set to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching # is when data is stored in memory instead of fetching data each time. # See https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] multilevel: # The maximum number of concurrent asynchronous operations can occur # when backfilling cache items. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of items to backfill per asynchronous operation. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-backfill-items [max_backfill_items: \u003cint\u003e | default = 10000] # Size of each subrange that bucket object is split into for better # caching. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-size [subrange_size: \u003cint\u003e | default = 16000] # Maximum number of sub-GetRange requests that a single GetRange request # can be split into when fetching chunks. Zero or negative value = # unlimited number of sub-requests. # CLI flag: -blocks-storage.bucket-store.chunks-cache.max-get-range-requests [max_get_range_requests: \u003cint\u003e | default = 3] # TTL for caching object attributes for chunks. # CLI flag: -blocks-storage.bucket-store.chunks-cache.attributes-ttl [attributes_ttl: \u003cduration\u003e | default = 168h] # TTL for caching individual chunks subranges. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-ttl [subrange_ttl: \u003cduration\u003e | default = 24h] metadata_cache: # Backend for metadata cache, if not empty. Supported values: memcached, # redis, and '' (disable). # CLI flag: -blocks-storage.bucket-store.metadata-cache.backend [backend: \u003cstring\u003e | default = \"\"] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV # query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup # made after that). # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. # If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should # run. If more keys are specified, internally keys are split into # multiple batches and fetched concurrently, honoring the max # concurrency. If set to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching # is when data is stored in memory instead of fetching data each time. # See https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # How long to cache list of tenants in the bucket. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl [tenants_list_ttl: \u003cduration\u003e | default = 15m] # How long to cache list of blocks for each tenant. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenant-blocks-list-ttl [tenant_blocks_list_ttl: \u003cduration\u003e | default = 5m] # How long to cache list of chunks for a block. # CLI flag: -blocks-storage.bucket-store.metadata-cache.chunks-list-ttl [chunks_list_ttl: \u003cduration\u003e | default = 24h] # How long to cache information that block metafile exists. Also used for # user deletion mark file. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-exists-ttl [metafile_exists_ttl: \u003cduration\u003e | default = 2h] # How long to cache information that block metafile doesn't exist. Also # used for user deletion mark file. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-doesnt-exist-ttl [metafile_doesnt_exist_ttl: \u003cduration\u003e | default = 5m] # How long to cache content of the metafile. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-content-ttl [metafile_content_ttl: \u003cduration\u003e | default = 24h] # Maximum size of metafile content to cache in bytes. Caching will be # skipped if the content exceeds this size. This is useful to avoid # network round trip for large content if the configured caching backend # has an hard limit on cached items size (in this case, you should set # this limit to the same limit in the caching backend). # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-max-size-bytes [metafile_max_size_bytes: \u003cint\u003e | default = 1048576] # How long to cache attributes of the block metafile. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-attributes-ttl [metafile_attributes_ttl: \u003cduration\u003e | default = 168h] # How long to cache attributes of the block index. # CLI flag: -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl [block_index_attributes_ttl: \u003cduration\u003e | default = 168h] # How long to cache content of the bucket index. # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl [bucket_index_content_ttl: \u003cduration\u003e | default = 5m] # Maximum size of bucket index content to cache in bytes. Caching will be # skipped if the content exceeds this size. This is useful to avoid # network round trip for large content if the configured caching backend # has an hard limit on cached items size (in this case, you should set # this limit to the same limit in the caching backend). # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-max-size-bytes [bucket_index_max_size_bytes: \u003cint\u003e | default = 1048576] # Duration after which the blocks marked for deletion will be filtered out # while fetching blocks. The idea of ignore-deletion-marks-delay is to # ignore blocks that are marked for deletion with some delay. This ensures # store can still serve blocks that are meant to be deleted but do not have # a replacement yet. Default is 6h, half of the default value for # -compactor.deletion-delay. # CLI flag: -blocks-storage.bucket-store.ignore-deletion-marks-delay [ignore_deletion_mark_delay: \u003cduration\u003e | default = 6h] # The blocks created since `now() - ignore_blocks_within` will not be # synced. This should be used together with `-querier.query-store-after` to # filter out the blocks that are too new to be queried. A reasonable value # for this flag would be `-querier.query-store-after - # blocks-storage.bucket-store.bucket-index.max-stale-period` to give some # buffer. 0 to disable. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: \u003cduration\u003e | default = 0s] bucket_index: # True to enable querier and store-gateway to discover blocks in the # storage via bucket index instead of bucket scanning. # CLI flag: -blocks-storage.bucket-store.bucket-index.enabled [enabled: \u003cboolean\u003e | default = false] # How frequently a bucket index, which previously failed to load, should # be tried to load again. This option is used only by querier. # CLI flag: -blocks-storage.bucket-store.bucket-index.update-on-error-interval [update_on_error_interval: \u003cduration\u003e | default = 1m] # How long a unused bucket index should be cached. Once this timeout # expires, the unused bucket index is removed from the in-memory cache. # This option is used only by querier. # CLI flag: -blocks-storage.bucket-store.bucket-index.idle-timeout [idle_timeout: \u003cduration\u003e | default = 1h] # The maximum allowed age of a bucket index (last updated) before queries # start failing because the bucket index is too old. The bucket index is # periodically updated by the compactor, while this check is enforced in # the querier (at query time). # CLI flag: -blocks-storage.bucket-store.bucket-index.max-stale-period [max_stale_period: \u003cduration\u003e | default = 1h] # One of concurrent, recursive, bucket_index. When set to concurrent, stores # will concurrently issue one call per directory to discover active blocks # in the bucket. The recursive strategy iterates through all objects in the # bucket, recursively traversing into each directory. This avoids N+1 calls # at the expense of having slower bucket iterations. bucket_index strategy # can be used in Compactor only and utilizes the existing bucket index to # fetch block IDs to sync. This avoids iterating the bucket but can be # impacted by delays of cleaner creating bucket index. # CLI flag: -blocks-storage.bucket-store.block-discovery-strategy [block_discovery_strategy: \u003cstring\u003e | default = \"concurrent\"] # Max size - in bytes - of a chunks pool, used to reduce memory allocations. # The pool is shared across all tenants. 0 to disable the limit. # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes [max_chunk_pool_bytes: \u003cint\u003e | default = 2147483648] # If enabled, store-gateway will lazily memory-map an index-header only once # required by a query. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-enabled [index_header_lazy_loading_enabled: \u003cboolean\u003e | default = false] # If index-header lazy loading is enabled and this setting is \u003e 0, the # store-gateway will release memory-mapped index-headers after 'idle # timeout' inactivity. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout [index_header_lazy_loading_idle_timeout: \u003cduration\u003e | default = 20m] # If true, Store Gateway will estimate postings size and try to lazily # expand postings if it downloads less data than expanding all postings. # CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled [lazy_expanded_postings_enabled: \u003cboolean\u003e | default = false] # Mark posting group as lazy if it fetches more keys than R * max series the # query should fetch. With R set to 100, a posting group which fetches 100K # keys will be marked as lazy if the current query only fetches 1000 series. # This config is only valid if lazy expanded posting is enabled. 0 disables # the limit. # CLI flag: -blocks-storage.bucket-store.lazy-expanded-posting-group-max-key-series-ratio [lazy_expanded_posting_group_max_key_series_ratio: \u003cfloat\u003e | default = 100] # Controls how many series to fetch per batch in Store Gateway. Default # value is 10000. # CLI flag: -blocks-storage.bucket-store.series-batch-size [series_batch_size: \u003cint\u003e | default = 10000] token_bucket_bytes_limiter: # Token bucket bytes limiter mode. Supported values are: disabled, dryrun, # enabled # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.mode [mode: \u003cstring\u003e | default = \"disabled\"] # Instance token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.instance-token-bucket-size [instance_token_bucket_size: \u003cint\u003e | default = 859832320] # User token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.user-token-bucket-size [user_token_bucket_size: \u003cint\u003e | default = 644874240] # Request token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.request-token-bucket-size [request_token_bucket_size: \u003cint\u003e | default = 4194304] tsdb: # Local directory to store TSDBs in the ingesters. # CLI flag: -blocks-storage.tsdb.dir [dir: \u003cstring\u003e | default = \"tsdb\"] # TSDB blocks range period. # CLI flag: -blocks-storage.tsdb.block-ranges-period [block_ranges_period: \u003clist of duration\u003e | default = 2h0m0s] # TSDB blocks retention in the ingester before a block is removed. This # should be larger than the block_ranges_period and large enough to give # store-gateways and queriers enough time to discover newly uploaded blocks. # CLI flag: -blocks-storage.tsdb.retention-period [retention_period: \u003cduration\u003e | default = 6h] # How frequently the TSDB blocks are scanned and new ones are shipped to the # storage. 0 means shipping is disabled. # CLI flag: -blocks-storage.tsdb.ship-interval [ship_interval: \u003cduration\u003e | default = 1m] # Maximum number of tenants concurrently shipping blocks to the storage. # CLI flag: -blocks-storage.tsdb.ship-concurrency [ship_concurrency: \u003cint\u003e | default = 10] # How frequently does Cortex try to compact TSDB head. Block is only created # if data covers smallest block range. Must be greater than 0 and max 30 # minutes. Note that up to 50% jitter is added to the value for the first # compaction to avoid ingesters compacting concurrently. # CLI flag: -blocks-storage.tsdb.head-compaction-interval [head_compaction_interval: \u003cduration\u003e | default = 1m] # Maximum number of tenants concurrently compacting TSDB head into a new # block # CLI flag: -blocks-storage.tsdb.head-compaction-concurrency [head_compaction_concurrency: \u003cint\u003e | default = 5] # If TSDB head is idle for this duration, it is compacted. Note that up to # 25% jitter is added to the value to avoid ingesters compacting # concurrently. 0 means disabled. # CLI flag: -blocks-storage.tsdb.head-compaction-idle-timeout [head_compaction_idle_timeout: \u003cduration\u003e | default = 1h] # The write buffer size used by the head chunks mapper. Lower values reduce # memory utilisation on clusters with a large number of tenants at the cost # of increased disk I/O operations. # CLI flag: -blocks-storage.tsdb.head-chunks-write-buffer-size-bytes [head_chunks_write_buffer_size_bytes: \u003cint\u003e | default = 4194304] # The number of shards of series to use in TSDB (must be a power of 2). # Reducing this will decrease memory footprint, but can negatively impact # performance. # CLI flag: -blocks-storage.tsdb.stripe-size [stripe_size: \u003cint\u003e | default = 16384] # Deprecated (use blocks-storage.tsdb.wal-compression-type instead): True to # enable TSDB WAL compression. # CLI flag: -blocks-storage.tsdb.wal-compression-enabled [wal_compression_enabled: \u003cboolean\u003e | default = false] # TSDB WAL type. Supported values are: 'snappy', 'zstd' and '' (disable # compression) # CLI flag: -blocks-storage.tsdb.wal-compression-type [wal_compression_type: \u003cstring\u003e | default = \"\"] # TSDB WAL segments files max size (bytes). # CLI flag: -blocks-storage.tsdb.wal-segment-size-bytes [wal_segment_size_bytes: \u003cint\u003e | default = 134217728] # True to flush blocks to storage on shutdown. If false, incomplete blocks # will be reused after restart. # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown [flush_blocks_on_shutdown: \u003cboolean\u003e | default = false] # If TSDB has not received any data for this duration, and all blocks from # TSDB have been shipped, TSDB is closed and deleted from local disk. If set # to positive value, this value should be equal or higher than # -querier.query-ingesters-within flag to make sure that TSDB is not closed # prematurely, which could cause partial query results. 0 or negative value # disables closing of idle TSDB. # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout [close_idle_tsdb_timeout: \u003cduration\u003e | default = 0s] # The size of the in-memory queue used before flushing chunks to the disk. # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: \u003cint\u003e | default = 0] # limit the number of concurrently opening TSDB's on startup # CLI flag: -blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup [max_tsdb_opening_concurrency_on_startup: \u003cint\u003e | default = 10] # Deprecated, use maxExemplars in limits instead. If the MaxExemplars value # in limits is set to zero, cortex will fallback on this value. This setting # enables support for exemplars in TSDB and sets the maximum number that # will be stored. 0 or less means disabled. # CLI flag: -blocks-storage.tsdb.max-exemplars [max_exemplars: \u003cint\u003e | default = 0] # True to enable snapshotting of in-memory TSDB data on disk when shutting # down. # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown [memory_snapshot_on_shutdown: \u003cboolean\u003e | default = false] # [EXPERIMENTAL] Configures the maximum number of samples per chunk that can # be out-of-order. # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max [out_of_order_cap_max: \u003cint\u003e | default = 32] # [EXPERIMENTAL] True to enable native histogram. # CLI flag: -blocks-storage.tsdb.enable-native-histograms [enable_native_histograms: \u003cboolean\u003e | default = false] # [EXPERIMENTAL] If enabled, ingesters will cache expanded postings when # querying blocks. Caching can be configured separately for the head and # compacted blocks. expanded_postings_cache: # If enabled, ingesters will cache expanded postings for the head block. # Only queries with with an equal matcher for metric __name__ are cached. head: # Whether the postings cache is enabled or not # CLI flag: -blocks-storage.expanded_postings_cache.head.enabled [enabled: \u003cboolean\u003e | default = false] # Max bytes for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.head.max-bytes [max_bytes: \u003cint\u003e | default = 10485760] # TTL for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.head.ttl [ttl: \u003cduration\u003e | default = 10m] # If enabled, ingesters will cache expanded postings for the compacted # blocks. The cache is shared between all blocks. blocks: # Whether the postings cache is enabled or not # CLI flag: -blocks-storage.expanded_postings_cache.block.enabled [enabled: \u003cboolean\u003e | default = false] # Max bytes for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.block.max-bytes [max_bytes: \u003cint\u003e | default = 10485760] # TTL for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.block.ttl [ttl: \u003cduration\u003e | default = 10m] ","categories":"","description":"","excerpt":" The querier service handles queries using the PromQL query language. …","ref":"/docs/blocks-storage/querier/","tags":"","title":"Querier"},{"body":"The query auditor is a tool bundled in the Cortex repository, but not included in Docker images – this must be built from source. It’s primarily useful for those developing Cortex, but can be helpful to operators as well during certain scenarios (backend migrations come to mind).\nHow it works The query-audit tool performs a set of queries against two backends that expose the Prometheus read API. This is generally the query-frontend component of two Cortex deployments. It will then compare the differences in the responses to determine the average difference for each query. It does this by:\nEnsuring the resulting label sets match. For each label set, ensuring they contain the same number of samples as their pair from the other backend. For each sample, calculates their difference against it’s pair from the other backend/label set. Calculates the average diff per query from the above diffs. Limitations It currently only supports queries with Matrix response types.\nUse cases Correctness testing when working on the read path. Comparing results from different backends. Example Configuration control: host: http://localhost:8080/prometheus headers: \"X-Scope-OrgID\": 1234 test: host: http://localhost:8081/prometheus headers: \"X-Scope-OrgID\": 1234 queries: - query: 'sum(rate(container_cpu_usage_seconds_total[5m]))' start: 2019-11-25T00:00:00Z end: 2019-11-28T00:00:00Z step_size: 15m - query: 'sum(rate(container_cpu_usage_seconds_total[5m])) by (container_name)' start: 2019-11-25T00:00:00Z end: 2019-11-28T00:00:00Z step_size: 15m - query: 'sum(rate(container_cpu_usage_seconds_total[5m])) without (container_name)' start: 2019-11-25T00:00:00Z end: 2019-11-26T00:00:00Z step_size: 15m - query: 'histogram_quantile(0.9, sum(rate(cortex_cache_value_size_bytes_bucket[5m])) by (le, job))' start: 2019-11-25T00:00:00Z end: 2019-11-25T06:00:00Z step_size: 15m # two shardable legs - query: 'sum without (instance, job) (rate(cortex_query_frontend_queue_length[5m])) or sum by (job) (rate(cortex_query_frontend_queue_length[5m]))' start: 2019-11-25T00:00:00Z end: 2019-11-25T06:00:00Z step_size: 15m # one shardable leg - query: 'sum without (instance, job) (rate(cortex_cache_request_duration_seconds_count[5m])) or rate(cortex_cache_request_duration_seconds_count[5m])' start: 2019-11-25T00:00:00Z end: 2019-11-25T06:00:00Z step_size: 15m Example Output Under ideal circumstances, you’ll see output like the following:\n$ go run ./tools/query-audit/ -f config.yaml 0.000000% avg diff for: query: sum(rate(container_cpu_usage_seconds_total[5m])) series: 1 samples: 289 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-28 00:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: sum(rate(container_cpu_usage_seconds_total[5m])) by (container_name) series: 95 samples: 25877 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-28 00:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: sum(rate(container_cpu_usage_seconds_total[5m])) without (container_name) series: 4308 samples: 374989 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-26 00:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: histogram_quantile(0.9, sum(rate(cortex_cache_value_size_bytes_bucket[5m])) by (le, job)) series: 13 samples: 325 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-25 06:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: sum without (instance, job) (rate(cortex_query_frontend_queue_length[5m])) or sum by (job) (rate(cortex_query_frontend_queue_length[5m])) series: 21 samples: 525 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-25 06:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: sum without (instance, job) (rate(cortex_cache_request_duration_seconds_count[5m])) or rate(cortex_cache_request_duration_seconds_count[5m]) series: 942 samples: 23550 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-25 06:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: sum by (namespace) (predict_linear(container_cpu_usage_seconds_total[5m], 10)) series: 16 samples: 400 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-25 06:00:00 +0000 UTC step: 15m0s 0.000000% avg diff for: query: sum by (namespace) (avg_over_time((rate(container_cpu_usage_seconds_total[5m]))[10m:]) \u003e 1) series: 4 samples: 52 start: 2019-11-25 00:00:00 +0000 UTC end: 2019-11-25 01:00:00 +0000 UTC step: 5m0s ","categories":"","description":"","excerpt":"The query auditor is a tool bundled in the Cortex repository, but not …","ref":"/docs/operations/query-auditor/","tags":"","title":"Query Auditor (tool)"},{"body":"REWE digital builds the technology that drives the e-commerce, app, and food pickup and delivery services for one of Germany’s largest grocery chains, REWE. Like other companies involved in the food supply chain, REWE has seen demand spike during the Covid-19 pandemic. Thanks to its adoption of Cortex last year, the monitoring team has been able to ensure stability at growing scale.\nThe REWE digital subsidiary was started in 2014 to advance its parent company’s digital transformation, so “we have a rather modern tech stack,” says Cloud Platform Engineer Martin Schneppenheim. A lot of the platform is run on Kubernetes using GKE while some is still running on-prem using Nomad. Still, there were some challenges that arose from the Spotify tribe model that the organization adopted. Each of the four tribes at REWE digital – ECOM, FULFILLMENT, CONTENT and PLATFORM, and while the tribes mostly converged on the same technologies, he says, “each platform team has its own solution for, say, Kafka, for Prometheus, for Grafana.”\nPlus, over the past six years, the company has grown from 30 employees to around 600.\nWith this rapid growth, they realized by the end of 2018 that they needed a new solution. The tipping point came when they experienced some out-of-memory issues with Prometheus, he says, “and we had no idea why.”\nEach tribe had one Prometheus HA pair that was used by all the teams in the tribe. One of the tribes used one Prometheus pair which required 30-60 gigabytes of RAM per instance. “We still saw some random out-of-memory kills, and we believe it was because some queries were loading too many samples,” Schneppenheim says. “We had several platform teams doing basically the same thing, and we wanted to tackle such issues organization-wide.”\nSearching for a scalable monitoring solution The solution needed to support the Prometheus format, since all of REWE’s microservices had a Prometheus end point. And the team wanted to have trust in the project’s longevity. After considering M3, Victoria Metrics, and Thanos, the REWE digital team decided to go with Cortex. “Cortex had just been released as a CNCF project, and there were several developers from different companies,” he says. “That was another plus point for us.” The key selling point, he adds, was Cortex’s multi-tenant support, which also involves the different protection mechanisms built into Cortex to limit a tenant’s usage so that a single tenant doesn’t affect the performance for other tenants. “Every platform team was providing one Prometheus for their tribe,” Schneppenheim says, “and we wanted to move to something like a software-as-a-service approach, with just one team that provides Cortex, which can be used by all the teams within the company.”\nImplementation Implementation began with the Big Data tribe, which has since merged with the other tribes and has been the smallest in the company. “We already had Prometheus set up, and we just switched the data source from Prometheus to Cortex,” Schneppenheim says. “In the beginning it was just one dashboard where we switched the data sources, and later on we switched the data source for the whole tribe so that all dashboards used the Cortex data source by default, and the Prometheus deployment basically acted as remote writing Prometheus. We always had the chance to just switch back to the Prometheus, in case there were any failures, so there was not a big risk.”\nIn fact, things went smoothly, and a few months later, the ECOM tribe started writing metrics to Cortex. At the same time, the platform tribe decided to create one Grafana instance and use organizations to offer multi-tenancy. After that second migration, the tribe’s teams were able to migrate dashboards to the new Grafana instance, and then start querying against that data. By the end of the year, all the tribes will have migrated to Cortex and the Grafana instance.\nREWE digital adopted Cortex at “a very early stage,” Schneppenheim says. At first, “sometimes we had to read the code, because there was little documentation, but we were still confident that we took the right decision, because we got lots of support [from the community] in debugging some problems, which were usually misconfigurations.”\nHe points out that configuration has become simpler over the past year, with default values set in v1.0, and more documentation: “Things definitely became better.”\nResults Cortex’s horizontal scaling has proven to be crucial during the Covid-19 pandemic, when REWE’s grocery and food delivery services have seen extremely high demand. “Our primary focus was to ensure stability, so we had to scale, and we deployed more containers,” he says. “That meant we had way more metrics than before, on the one hand, and on the other hand, I believe our developers were watching our dashboards more closely, so we had way more queries as well.”\nSchneppenheim estimates that over the past two months, reads and writes have increased significantly, and the platform was able to handle the added load. Plus, “it was quite easy deploying another set of queriers,” he says.\nAside from that, Schneppenheim says, “the biggest advantage for our company is that we now have a team that can offer one thing as an internal service.” While the tribes’ ops teams still have to manage their own Prometheus servers, they have a much more stable and scalable system. The challenges are unpredictable resource usage on querying and some queries that can load too much data causing Prometheus to OOM, but with Cortex handling all the queries, this is no longer a problem. And while Schneppenheim’s team is still just two people (they’re hiring!), he adds, “we can spend more time actually learning how to run it, and become an expert within the company for Cortex and the things that come along with it, like high cardinality metric series, which we see every two weeks or so. We are the contact for all the monitoring now.”\nThere have been other technical advantages, too: “We have no gaps anymore in our Prometheus and Grafana,” he says. “In case a Prometheus instance fails or if it needs to be restarted, we automatically switch over to the replica with the HA tracker, which is a great thing.”\nWith Cortex’s query results cache, the queries are cached. The REWE digital team has found that this feature makes dashboards “super fast, because the query is likely already cached and it just has to load the new 30 seconds or so, since the last refresh,” says Schneppenheim. “Preloaded dashboards load or refresh really, really fast.”\nPlus, there is a higher retention with Cortex. “We now have 60 days’ retention; we used to have seven days only,” he says.\nThe benefits are also clear as the infrastructure grows. REWE digital has added a few more small Kubernetes clusters, which “obviously have the same monitoring/alerting needs as our biggest clusters,” he says. Previously, the team would have to deploy Prometheus and a separate Grafana instance (along with NGINX and DNS setup).\n“With our new SaaS approach, making monitoring available for these is as easy as adding a Prometheus pair, which sends metrics to our Cortex cluster, and adding this new tenant in our Grafana organization,” he says.\nWith Cortex, they’ve also been able to solve two use cases (Kubernetes clusters that had been split for technical reasons, and cloud migration) that required metrics from two different clusters to be available with the same tenant. “The dev teams had the need to aggregate metrics across these two clusters, which was easily possible, because we just ingested them under the same tenant ID,” says Schneppenheim.\nAnd those out-of-memory issues? “We are constantly growing, not only on the query side, but also on the ingesting metrics side as we onboard teams and tribes,” he says. “But we have fine-tuned it quite well, and there are not as many OOM kills, and if there are, we don’t see them in Grafana. That’s important to us, that our developers have a smooth experience.” (Most tribes use Grafana alerting; one uses Prometheus Alertmanager.)\nLooking ahead REWE digital’s main focus right now is to onboard the rest of the teams to Cortex. But looking ahead, the team is exploring Grafana Cloud Agent for the tribes that aren’t using Prometheus Alertmanager. “They don’t need Prometheus; we only use Prometheus to scrape the targets and send samples to our Cortex,” he says, “so that could definitely be interesting, especially given the performance improvements. Its sole purpose is to send metrics to Cortex as our remote-write backend, so maybe there will be other advantages in the future, like a more close monitoring.”\nThe in-development Cortex Blocks storage engine is also interesting to the team as a solution for the bottleneck it has around a small BigTable cluster. “We just run three BigTable nodes, and the BigTable read latency sometimes peaks at one second, which is also the upper bucket limit in the histogram,” he says. “This happens if users open Grafana dashboards querying a long time range with many panels. Our hope is that switching from BigTable to the new storage engine would fix this as the object store (GCS) scales on-demand.”\nThe REWE digital team has built and open sourced its own Cortex gateway, which is on the project roadmap. “This might be a chance for us to contribute,” says Schneppenheim.\nSchneppenheim is also hopeful that the positive results REWE digital has seen with Cortex may lead to its further adoption throughout the greater REWE Group organization. “We’re just a small company within the REWE group,” he says, but “we might offer it as internal software as a service for other parts of the Group. They can trust our solution.”\n","categories":"","description":"","excerpt":"REWE digital builds the technology that drives the e-commerce, app, …","ref":"/docs/case-studies/rewe-digital/","tags":"","title":"How Cortex helped REWE digital ensure stability while scaling services during the Covid-19 pandemic"},{"body":"The blocks storage is a Cortex storage engine based on Prometheus TSDB: it stores each tenant’s time series into their own TSDB which write out their series to a on-disk block (defaults to 2h block range periods). Each block is composed by chunk files - containing the timestamp-value pairs for multiple series - and an index, which indexes metric names and labels to time series in the chunk files.\nThe supported backends for the blocks storage are:\nAmazon S3 Google Cloud Storage Microsoft Azure Storage OpenStack Swift Local Filesystem (single node only) Internally, some components are based on Thanos, but no Thanos knowledge is required in order to run it.\nArchitecture When running the Cortex blocks storage, the Cortex architecture doesn’t significantly change and thus the general architecture documentation applies to the blocks storage as well. However, there are two additional Cortex services when running the blocks storage:\nStore-gateway Compactor The store-gateway is responsible to query blocks and is used by the querier at query time. The store-gateway is required when running the blocks storage.\nThe compactor is responsible to merge and deduplicate smaller blocks into larger ones, in order to reduce the number of blocks stored in the long-term storage for a given tenant and query them more efficiently. It also keeps the bucket index updated and, for this reason, it’s a required component.\nThe alertmanager and ruler components can also use object storage to store its configurations and rules uploaded by users. In that case a separate bucket should be created to store alertmanager configurations and rules: using the same bucket between ruler/alertmanager and blocks will cause issue with the compactor.\nThe write path Ingesters receive incoming samples from the distributors. Each push request belongs to a tenant, and the ingester appends the received samples to the specific per-tenant TSDB stored on the local disk. The received samples are both kept in-memory and written to a write-ahead log (WAL) and used to recover the in-memory series in case the ingester abruptly terminates. The per-tenant TSDB is lazily created in each ingester as soon as the first samples are received for that tenant.\nThe in-memory samples are periodically flushed to disk - and the WAL truncated - when a new TSDB block is created, which by default occurs every 2 hours. Each newly created block is then uploaded to the long-term storage and kept in the ingester until the configured -blocks-storage.tsdb.retention-period expires, in order to give queriers and store-gateways enough time to discover the new block on the storage and download its index-header.\nIn order to effectively use the WAL and being able to recover the in-memory series upon ingester abruptly termination, the WAL needs to be stored to a persistent disk which can survive in the event of an ingester failure (ie. AWS EBS volume or GCP persistent disk when running in the cloud). For example, if you’re running the Cortex cluster in Kubernetes, you may use a StatefulSet with a persistent volume claim for the ingesters. The location on the filesystem where the WAL is stored is the same where local TSDB blocks (compacted from head) are stored and cannot be decoupled. See also the timeline of block uploads and disk space estimate.\nDistributor series sharding and replication The series sharding and replication done by the distributor doesn’t change based on the storage engine.\nIt’s important to note that due to the replication factor N (typically 3), each time series is stored by N ingesters. Since each ingester writes its own block to the long-term storage, this leads a storage utilization N times more. Compactor solves this problem by merging blocks from multiple ingesters into a single block, and removing duplicated samples. After blocks compaction, the storage utilization is significantly smaller.\nFor more information, please refer to the following dedicated sections:\nCompactor Production tips The read path Queriers and store-gateways periodically iterate over the storage bucket to discover blocks recently uploaded by ingesters.\nFor each discovered block, queriers only download the block’s meta.json file (containing some metadata including min and max timestamp of samples within the block), while store-gateways download the meta.json as well as the index-header, which is a small subset of the block’s index used by the store-gateway to lookup series at query time.\nQueriers use the blocks metadata to compute the list of blocks that need to be queried at query time and fetch matching series from the store-gateway instances holding the required blocks.\nFor more information, please refer to the following dedicated sections:\nQuerier Store-gateway Production tips Known issues GitHub issues tagged with the storage/blocks label are the best source of currently known issues affecting the blocks storage.\n","categories":"","description":"","excerpt":"The blocks storage is a Cortex storage engine based on Prometheus …","ref":"/docs/blocks-storage/","tags":"","title":"Blocks Storage"},{"body":"Buoyant, the creator of Linkerd, has had a close relationship with Prometheus and related technologies for years. As of today, that relationship now includes Cortex.\nLinkerd is an open source service mesh for Kubernetes, and part of the CNCF, along with Prometheus and Cortex. Linkerd provides three key classes of features: observability, reliability, and security—all without requiring any changes to your code. That first pillar, observability, has motivated deep Prometheus integration, and is the focus of this case study.\nLinkerd x Prometheus In 2016, Linkerd 1.0 shipped with a visualization module built with Prometheus and Grafana. With the release of Linkerd 2.0 in 2018, Prometheus and Grafana were promoted to first-class components. That evolution was documented in a blog post entitled Prometheus the Right Way, and a KubeCon North America 2018 talk entitled Linkerd 2.0, Now with Extra Prometheus, both co-authored by Frederic Branczyk of Polar Signals and Andrew Seigner of Buoyant.\nThis deep integration in Linkerd 2.0 provided out-of-the-box golden metrics, that is, success rate, request rate, and latency, across all your Kubernetes workloads.\nLinkerd 2.0 with integrated Prometheus and Grafana, out-of-the-box golden metrics\nBuoyant x Cortex Building on the success of Linkerd, Buoyant has created Buoyant Cloud, a global platform health dashboard for Kubernetes. Leveraging Linkerd’s integration with Prometheus, Buoyant Cloud rolls up Linkerd metrics across all Kubernetes clusters to provide global platform-wide observability, including advanced features such as cross-cluster Service Level Objectives for success rate and latency.\nService Level Objectives in Buoyant Cloud\nBuoyant Cloud Prototyping with Prometheus To enable observability in Buoyant Cloud, customers install an agent into their Kubernetes clusters. That agent gathers metrics from Linkerd and sends them up to Buoyant Cloud. Early prototypes of Buoyant Cloud received these metrics, wrote them to pushgateway, and then our own larger Prometheus would scrape metrics from there. While this had the nice property that Linkerd metrics looked the same in Buoyant Cloud as they did in individual Linkerd clusters, gathering metrics across all customers into a single Prometheus instance created an inherent scaling limitation, and a single-point-of-failure.\nObservability Requirements Thinking beyond our early prototypes, we came up with four core requirements for our observability system:\nScalable - To support all Kubernetes clusters across our growing customer base, we needed a system that could scale as-needed. Bonus points if we could scale our reads and writes independently. Reliable - Users of Linkerd expect nothing less. Multi-tenant - To provide an extra layer of security and performance isolation between customers. Prometheus-compatible - To give our customers a familiar API, and to allow a drop-in replacement for our prototype Prometheus instance. Those first two requirements in particular are not compatible with a single instance of anything. Even Prometheus, which does exceptionally well as a single instance, is intentionally not designed as a distributed, scalable system.\nTake a look at those requirements again, then go to the Cortex website:\nhttps://cortexmetrics.io/\nThis was not intentional, but it was a fortuitous coincidence. Cortex’s one-line description of itself literally satisfied all four of our requirements for Buoyant Cloud.\nBuoyant Cloud Production-Ready with Cortex We set about building a proof of concept to validate whether Cortex could be a viable replacement for Buoyant Cloud’s prototype Prometheus-based observability.\nDevelopment environment The very first attempt at Cortex integration in our development environment was surprisingly smooth, thanks largely to Cortex’s single instance, single process mode. This enabled our developers to operate all of Cortex’s services as a single process in our docker-compose development environment, with no dependencies.\n(Almost) a drop-in replacement Cortex provides a Prometheus-compatible API. To enable multi-tenancy, you must set a X-Scope-OrgID header on every request with a unique identifier for the tenant. We already had a unique public identifier for each Buoyant Cloud customer, so that was a natural fit for this header. Modifying all reads and writes to Prometheus to set this header was a relatively straightforward code change, considering we were about to completely swap out a back-end storage system.\nBlocks Storage Being the creators of Linkerd, we care a lot about operational complexity. Thus, it was no surprise that we wanted to minimize operational complexity and cost for ourselves in maintaining Buoyant Cloud. Cortex’s Chunk Storage was a concern here. Operating our own Cassandra cluster would incur developer time, and paying for DynamoDB, BigTable, or Azure Cosmos DB would incur cost. Cortex’s Blocks Storage removed this complexity and cost by relying on a simple and cheap object store, such as S3, GCS, or Azure Storage. At the time, Blocks Storage was still marked experimental. That’s when we hopped into a Q\u0026A at a KubeCon EU 2020 talk, Scaling Prometheus: How We Got Some Thanos Into Cortex by Thor Hansen of HashiCorp and Marco Pracucci of Grafana Labs. We asked the Cortex maintainers how close to General Availability they felt Blocks Storage was. While wisely not guaranteeing a date, they hinted it may be ready by the next Cortex release (and that hint turned out to be true, Blocks Storage was marked stable in Cortex 1.4.0!). This gave us enough confidence to build our proof of concept around Blocks Storage.\nRecording Rules Buoyant Cloud uses recording rules for important success rate and latency metrics. One unexpected challenge was that Cortex’s multi-tenancy applies to recording rules. This meant that for every customer, we needed to push an identical set of recording rules to the Cortex Ruler API. We ended up writing a small background process that continually checks our database for new customers, and pushes recording rules when they appear.\nCortex in Prod: Writes, then reads Once all the pieces were in place, we enabled production writes to our new Cortex cluster. We continued writing to and reading from our existing Prometheus. Writing to Cortex and Prometheus simultaneously enabled validation of three things. We could 1) Evaluate Cortex’s read performance under full production write load, 2) ensure Cortex query results matched Prometheus, and 3) gather enough historical metrics in Cortex to minimize data loss for our customers.\nWhen we were satisfied with all three of these validation steps, we switched all reads to Cortex. This was the smallest code change in the entire migration. We simply swapped out a few command-line flags pointing to prometheus:9009 with query-frontend.cortex.svc.cluster.local:9009/api/prom. Boom! It all worked!\nThe moment we turned on Cortex reads in production (numbers do not reflect full production load).\nLooking Ahead Our mission with Buoyant Cloud is to enable our customers to build observable, scalable, and secure Kubernetes platforms, without the complexity of stitching together lots of components.\nUsers of Linkerd expect their observability to “just work”, and expect it to be something they are already familiar with and have tools to integrate with. For Linkerd running on a single Kubernetes cluster, Prometheus fits the bill perfectly. For Buoyant Cloud, we believe Cortex can deliver that same familiarity and integration story for all Kubernetes clusters across all customers.\n","categories":"","description":"","excerpt":"Buoyant, the creator of Linkerd, has had a close relationship with …","ref":"/docs/case-studies/buoyant-cloud/","tags":"","title":"Buoyant Cloud and Cortex: Standing on the shoulders of Linkerd and Prometheus"},{"body":"The Cortex documentation is compiled into a website published at cortexmetrics.io. These instructions explain how to run the website locally, in order to have a quick feedback loop while contributing to the documentation or website styling.\nInitial setup The following initial setup is required only once:\nInstall Hugo (extended version) Look for value of HUGO_VERSION in build-image/Dockerfile. Install Node.js v14 or above (alternatively via nvm) Install required Node modules with: cd website \u0026\u0026 npm install \u0026\u0026 cd - Install embedmd v1.0.0: go install github.com/campoy/embedmd@v1.0.0 Run make BUILD_IN_CONTAINER=false web-build Run it Once the initial setup is completed, you can run the website with the following command. The local website will run at http://localhost:1313/\n# Keep this running make web-serve Whenever you change the content of docs/ or markdown files in the repository root / you should run:\nmake BUILD_IN_CONTAINER=false web-pre Whenever you change the config file or CLI flags in the Cortex code, you should rebuild the config file reference documentation:\nmake BUILD_IN_CONTAINER=false doc web-pre ","categories":"","description":"","excerpt":"The Cortex documentation is compiled into a website published at …","ref":"/docs/contributing/how-to-run-the-website-locally/","tags":"","title":"How to run the website locally"},{"body":"You can use the Cortex query frontend with any Prometheus-API compatible service, including Prometheus and Thanos. Use this config file to get the benefits of query parallelisation and caching.\n# Disable the requirement that every request to Cortex has a # X-Scope-OrgID header. `fake` will be substituted in instead. auth_enabled: false # We only want to run the query-frontend module. target: query-frontend # We don't want the usual /api/prom prefix. http_prefix: server: http_listen_port: 9091 query_range: split_queries_by_interval: 24h align_queries_with_step: true cache_results: true # list of request headers forwarded by query frontend to downstream queriers. forward_headers_list: - Authorization results_cache: cache: # We're going to use the in-process \"FIFO\" cache, but you can enable # memcached below. enable_fifocache: true fifocache: size: 1024 validity: 24h # If you want to use a memcached cluster, you can either configure a # headless service in Kubernetes and Cortex will discover the individual # instances using a SRV DNS query (host) or list comma separated # memcached addresses. # host + service: this is the config you should set when you use the # SRV DNS (this is considered stable) # addresses: this is experimental and supports service discovery # (https://cortexmetrics.io/docs/configuration/arguments/#dns-service-discovery) # so it could either be a list of single addresses, or a SRV record # prefixed with dnssrvnoa+. Cortex will then do client-side hashing to # spread the load evenly. # memcached: # expiration : 24h # memcached_client: # host: memcached.default.svc.cluster.local # service: memcached # addresses: \"\" # consistent_hash: true frontend: log_queries_longer_than: 1s compress_responses: true # The query endpoint URL of a Prometheus-API compatible service to which the query-frontend should connect to. downstream_url: http://prometheus.mydomain.com ","categories":"","description":"","excerpt":"You can use the Cortex query frontend with any Prometheus-API …","ref":"/docs/configuration/prometheus-frontend/","tags":"","title":"Prometheus Frontend"},{"body":"The query-tee is a standalone service which can be used for testing purposes to compare the query performances of 2+ backend systems (ie. Cortex clusters) ingesting the same exact series.\nThis service exposes Prometheus-compatible read API endpoints and, for each received request, performs the request against all backends tracking the response time of each backend and then sends back to the client one of the received responses.\nHow to run it You can run query-tee in two ways:\nBuild it from sources go run ./cmd/query-tee -help Run it via the provided Docker image docker run quay.io/cortexproject/query-tee -help The service requires at least 1 backend endpoint (but 2 are required in order to compare performances) configured as comma-separated HTTP(S) URLs via the CLI flag -backend.endpoints. For each incoming request, query-tee will clone the request and send it to each backend, tracking performance metrics for each backend before sending back the response to the client.\nHow it works API endpoints The following Prometheus API endpoints are supported by query-tee:\n/api/v1/query (GET) /api/v1/query_range (GET) /api/v1/labels (GET) /api/v1/label/{name}/values (GET) /api/v1/series (GET) /api/v1/metadata (GET) /api/v1/alerts (GET) /api/v1/rules (GET) Pass-through requests query-tee supports acting as a transparent proxy for requests to routes not matching any of the documented API endpoints above. When enabled, those requests are passed on to just the configured preferred backend. To activate this feature it requires setting -proxy.passthrough-non-registered-routes=true flag and configuring a preferred backend.\nAuthentication query-tee supports HTTP basic authentication. It allows either to configure username and password in the backend URL, to forward the request auth to the backend or merge the two.\nThe request sent from the query-tee to the backend includes HTTP basic authentication when one of the following conditions are met:\nIf the endpoint URL has username and password, query-tee uses it. If the endpoint URL has username only, query-tee keeps the username and inject the password received in the incoming request (if any). If the endpoint URL has no username and no password, query-tee forwards the incoming request basic authentication (if any). Backend response selection query-tee allows to configure a preferred backend from which picking the response to send back to the client. The preferred backend can be configured via the CLI flag -backend.preferred=\u003chostname\u003e, setting it to the hostname of the preferred backend.\nWhen a preferred backend is set, query-tee sends back to the client:\nThe preferred backend response if the status code is 2xx or 4xx Otherwise, the first received 2xx or 4xx response if at least a backend succeeded Otherwise, the first received response When a preferred backend is not set, query-tee sends back to the client:\nThe first received 2xx or 4xx response if at least a backend succeeded Otherwise, the first received response Note: from the query-tee perspective, a backend request is considered successful even if the status code is 4xx because it generally means the error is due to an invalid request and not to a backend issue.\nBackend results comparison query-tee allows to optionally enable the query results comparison between two backends. The results comparison can be enabled via the CLI flag -proxy.compare-responses=true and requires exactly two configured backends with a preferred one.\nWhen the comparison is enabled, the query-tee compares the response received from the two configured backends and logs a message for each query whose results don’t match, as well as keeps track of the number of successful and failed comparison through the metric cortex_querytee_responses_compared_total.\nFloating point sample values are compared with a small tolerance that can be configured via -proxy.value-comparison-tolerance. This prevents false positives due to differences in floating point values rounding introduced by the non deterministic series ordering within the Prometheus PromQL engine.\nSlow backends query-tee sends back to the client the first viable response as soon as available, without waiting to receive a response from all backends.\nExported metrics query-tee exposes the following Prometheus metrics on the port configured via the CLI flag -server.metrics-port:\n# HELP cortex_querytee_request_duration_seconds Time (in seconds) spent serving HTTP requests. # TYPE cortex_querytee_request_duration_seconds histogram cortex_querytee_request_duration_seconds_bucket{backend=\"\u003chostname\u003e\",method=\"\u003cmethod\u003e\",route=\"\u003croute\u003e\",status_code=\"\u003cstatus\u003e\",le=\"\u003cbucket\u003e\"} cortex_querytee_request_duration_seconds_sum{backend=\"\u003chostname\u003e\",method=\"\u003cmethod\u003e\",route=\"\u003croute\u003e\",status_code=\"\u003cstatus\u003e\"} cortex_querytee_request_duration_seconds_count{backend=\"\u003chostname\u003e\",method=\"\u003cmethod\u003e\",route=\"\u003croute\u003e\",status_code=\"\u003cstatus\u003e\"} # HELP cortex_querytee_responses_total Total number of responses sent back to the client by the selected backend. # TYPE cortex_querytee_responses_total counter cortex_querytee_responses_total{backend=\"\u003chostname\u003e\",method=\"\u003cmethod\u003e\",route=\"\u003croute\u003e\"} # HELP cortex_querytee_responses_compared_total Total number of responses compared per route name by result. # TYPE cortex_querytee_responses_compared_total counter cortex_querytee_responses_compared_total{route=\"\u003croute\u003e\",result=\"\u003csuccess|fail\u003e\"} ","categories":"","description":"","excerpt":"The query-tee is a standalone service which can be used for testing …","ref":"/docs/operations/query-tee/","tags":"","title":"Query Tee (service)"},{"body":"Because Cortex is designed to run multiple instances of each component (ingester, querier, etc.), you probably want to automate the placement and shepherding of these instances. Most users choose Kubernetes to do this, but this is not mandatory.\nConfiguration Resource requests If using Kubernetes, each container should specify resource requests so that the scheduler can place them on a node with sufficient capacity.\nFor example, an ingester might request:\nresources: requests: cpu: 4 memory: 10Gi The specific values here should be adjusted based on your own experiences running Cortex - they are very dependent on the rate of data arriving and other factors such as series churn.\nTake extra care with ingesters Ingesters hold hours of timeseries data in memory; you can configure Cortex to replicate the data, but you should take steps to avoid losing all replicas at once:\nDon’t run multiple ingesters on the same node. Don’t run ingesters on preemptible/spot nodes. Spread out ingesters across racks / availability zones / whatever applies in your datacenters. You can ask Kubernetes to avoid running on the same node like this:\naffinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - key: name operator: In values: - ingester topologyKey: \"kubernetes.io/hostname\" Give plenty of time for an ingester to hand over or flush data to store when shutting down; for Kubernetes, this looks like:\nterminationGracePeriodSeconds: 2400 Ask Kubernetes to limit rolling updates to one ingester at a time and signal the old one to stop before the new one is ready:\nstrategy: rollingUpdate: maxSurge: 0 maxUnavailable: 1 Ingesters provide an HTTP hook to signal readiness when all is well; this is valuable because it stops a rolling update at the first problem:\nreadinessProbe: httpGet: path: /ready port: 80 We do not recommend configuring a liveness probe on ingesters - killing them is a last resort and should not be left to a machine.\n","categories":"","description":"","excerpt":"Because Cortex is designed to run multiple instances of each component …","ref":"/docs/guides/running-cortex-on-kubernetes/","tags":"","title":"Running Cortex on Kubernetes"},{"body":" The store-gateway is the Cortex service responsible to query series from blocks. The store-gateway is required when running the Cortex blocks storage.\nThe store-gateway is semi-stateful.\nHow it works The store-gateway needs to have an almost up-to-date view over the storage bucket, in order to discover blocks belonging to their shard. The store-gateway can keep the bucket view updated in to two different ways:\nPeriodically scanning the bucket (default) Periodically downloading the bucket index Bucket index disabled (default) At startup store-gateways iterate over the entire storage bucket to discover blocks for all tenants and download the meta.json and index-header for each block. During this initial bucket synchronization phase, the store-gateway /ready readiness probe endpoint will fail.\nWhile running, store-gateways periodically rescan the storage bucket to discover new blocks (uploaded by the ingesters and compactor) and blocks marked for deletion or fully deleted since the last scan (as a result of compaction). The frequency at which this occurs is configured via -blocks-storage.bucket-store.sync-interval.\nThe blocks chunks and the entire index are never fully downloaded by the store-gateway. The index-header is stored to the local disk, in order to avoid to re-download it on subsequent restarts of a store-gateway. For this reason, it’s recommended - but not required - to run the store-gateway with a persistent disk. For example, if you’re running the Cortex cluster in Kubernetes, you may use a StatefulSet with a persistent volume claim for the store-gateways.\nFor more information about the index-header, please refer to Binary index-header documentation.\nBucket index enabled When bucket index is enabled, the overall workflow is the same but, instead of iterating over the bucket objects, the store-gateway fetch the bucket index for each tenant belonging to their shard in order to discover each tenant’s blocks and block deletion marks.\nFor more information about the bucket index, please refer to bucket index documentation.\nBlocks sharding and replication The store-gateway optionally supports blocks sharding. Sharding can be used to horizontally scale blocks in a large cluster without hitting any vertical scalability limit.\nWhen sharding is enabled, store-gateway instances builds an hash ring and blocks get sharded and replicated across the pool of store-gateway instances registered within the ring.\nStore-gateways continuously monitor the ring state and whenever the ring topology changes (e.g. a new instance has been added/removed or gets healthy/unhealthy) each store-gateway instance resync the blocks assigned to its shard, based on the block ID hash matching the token ranges assigned to the instance itself within the ring.\nFor each block belonging to a store-gateway shard, the store-gateway loads its meta.json, the deletion-mark.json and the index-header. Once a block is loaded on the store-gateway, it’s ready to be queried by queriers. When the querier queries blocks through a store-gateway, the response will contain the list of actually queried block IDs. If a querier tries to query a block which has not been loaded by a store-gateway, the querier will either retry on a different store-gateway (if blocks replication is enabled) or fail the query.\nBlocks can be replicated across multiple store-gateway instances based on a replication factor configured via -store-gateway.sharding-ring.replication-factor. The blocks replication is used to protect from query failures caused by some blocks not loaded by any store-gateway instance at a given time like, for example, in the event of a store-gateway failure or while restarting a store-gateway instance (e.g. during a rolling update).\nThis feature can be enabled via -store-gateway.sharding-enabled=true and requires the backend hash ring to be configured via -store-gateway.sharding-ring.* flags (or their respective YAML config options).\nSharding strategies The store-gateway supports two sharding strategies:\ndefault shuffle-sharding zone-stable-shuffle-sharding The default sharding strategy spreads the blocks of each tenant across all store-gateway instances. It’s the easiest form of sharding supported, but doesn’t provide any workload isolation between different tenants.\nThe shuffle-sharding strategy spreads the blocks of a tenant across a subset of store-gateway instances. This way, the number of store-gateway instances loading blocks of a single tenant is limited and the blast radius of any issue that could be introduced by the tenant’s workload is limited to its shard instances.\nThe shuffle sharding strategy can be enabled via -store-gateway.sharding-strategy=shuffle-sharding and requires the -store-gateway.tenant-shard-size flag (or their respective YAML config options) to be set to the default shard size, which is the default number of store-gateway instances each tenant should be sharded to. The shard size can then be overridden on a per-tenant basis setting the store_gateway_tenant_shard_size in the limits overrides.\nThe zone-stable-shuffle-sharding strategy achieves the same as the shuffle-sharding strategy, but using a different sharding algorithm. The new sharding algorithm ensures that when zone awareness is enabled, when shard size increases or decreases by one, the replicas for any block should only change at most by one instance. This is important for querying store gateway because a block can be retried at most 3 times.\nZone stable shuffle sharding can be enabled via -store-gateway.sharding-ring.zone-stable-shuffle-sharding CLI flag.\nIt will become the default shuffle sharding strategy for store gateway in v1.17.0 release and the previous shuffle sharding algorithm will be removed in v1.18.0 release.\nPlease check out the shuffle sharding documentation for more information about how it works.\nAuto-forget When a store-gateway instance cleanly shutdowns, it automatically unregisters itself from the ring. However, in the event of a crash or node failure, the instance will not be unregistered from the ring, potentially leaving a spurious entry in the ring forever.\nTo protect from this, when an healthy store-gateway instance finds another instance in the ring which is unhealthy for more than 10 times the configured -store-gateway.sharding-ring.heartbeat-timeout, the healthy instance forcibly removes the unhealthy one from the ring.\nThis feature is called auto-forget and is built into the store-gateway.\nZone-awareness The store-gateway replication optionally supports zone-awareness. When zone-aware replication is enabled and the blocks replication factor is \u003e 1, each block is guaranteed to be replicated across store-gateway instances running in different availability zones.\nTo enable the zone-aware replication for the store-gateways you should:\nConfigure the availability zone for each store-gateway via the -store-gateway.sharding-ring.instance-availability-zone CLI flag (or its respective YAML config option) Enable blocks zone-aware replication via the -store-gateway.sharding-ring.zone-awareness-enabled CLI flag (or its respective YAML config option). Please be aware this configuration option should be set to store-gateways, queriers and rulers. Rollout store-gateways, queriers and rulers to apply the new configuration Waiting for stable ring at startup In the event of a cluster cold start or scale up of 2+ store-gateway instances at the same time we may end up in a situation where each new store-gateway instance starts at a slightly different time and thus each one runs the initial blocks sync based on a different state of the ring. For example, in case of a cold start, the first store-gateway joining the ring may load all blocks since the sharding logic runs based on the current state of the ring, which is 1 single store-gateway.\nTo reduce the likelihood this could happen, the store-gateway waits for a stable ring at startup. A ring is considered stable if no instance is added/removed to the ring for at least -store-gateway.sharding-ring.wait-stability-min-duration. If the ring keep getting changed after -store-gateway.sharding-ring.wait-stability-max-duration, the store-gateway will stop waiting for a stable ring and will proceed starting up normally.\nTo disable this waiting logic, you can start the store-gateway with -store-gateway.sharding-ring.wait-stability-min-duration=0.\nBlocks index-header The index-header is a subset of the block index which the store-gateway downloads from the object storage and keeps on the local disk in order to speed up queries.\nAt startup, the store-gateway downloads the index-header of each block belonging to its shard. A store-gateway is not ready until this initial index-header download is completed. Moreover, while running, the store-gateway periodically looks for newly uploaded blocks in the storage and downloads the index-header for the blocks belonging to its shard.\nIndex-header lazy loading By default, each index-header is memory mapped by the store-gateway right after downloading it. In a cluster with a large number of blocks, each store-gateway may have a large amount of memory mapped index-headers, regardless how frequently they’re used at query time.\nCortex supports a configuration option -blocks-storage.bucket-store.index-header-lazy-loading-enabled=true to enable index-header lazy loading. When enabled, index-headers will be memory mapped only once required by a query and will be automatically released after -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout time of inactivity.\nCaching The store-gateway supports the following caches:\nIndex cache Chunks cache Metadata cache Caching is optional, but highly recommended in a production environment. Please also check out the production tips for more information about configuring the cache.\nIndex cache The store-gateway can use a cache to speed up lookups of postings and series from TSDB blocks indexes. Two backends are supported:\ninmemory memcached redis In-memory index cache The inmemory index cache is enabled by default and its max size can be configured through the flag -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes (or config file). The trade-off of using the in-memory index cache is:\nPros: zero latency Cons: increased store-gateway memory usage, not shared across multiple store-gateway replicas (when sharding is disabled or replication factor \u003e 1) Memcached index cache The memcached index cache allows to use Memcached as cache backend. This cache backend is configured using -blocks-storage.bucket-store.index-cache.backend=memcached and requires the Memcached server(s) addresses via -blocks-storage.bucket-store.index-cache.memcached.addresses (or config file). The addresses are resolved using the DNS service provider.\nThe trade-off of using the Memcached index cache is:\nPros: can scale beyond a single node memory (Memcached cluster), shared across multiple store-gateway instances Cons: higher latency in the cache round trip compared to the in-memory one The Memcached client uses a jump hash algorithm to shard cached entries across a cluster of Memcached servers. For this reason, you should make sure memcached servers are not behind any kind of load balancer and their address is configured so that servers are added/removed to the end of the list whenever a scale up/down occurs.\nFor example, if you’re running Memcached in Kubernetes, you may:\nDeploy your Memcached cluster using a StatefulSet Create a headless service for Memcached StatefulSet Configure the Cortex’s Memcached client address using the dnssrvnoa+ service discovery Redis index cache The redis index cache allows to use Redis as cache backend. This cache backend is configured using -blocks-storage.bucket-store.index-cache.backend=redis and requires the Redis server(s) addresses via -blocks-storage.bucket-store.index-cache.redis.addresses (or config file).\nUsing redis as the cache backend has similar trade-offs as using memcached cache backend. However, client side caching can be enabled when using redis backend to avoid Store Gateway fetching data from cache each time. See here for more info and it can be enabled by setting flag -blocks-storage.bucket-store.index-cache.redis.cache-size \u003e 0.\nChunks cache Store-gateway can also use a cache for storing chunks fetched from the storage. Chunks contain actual samples, and can be reused if user query hits the same series for the same time range.\nTo enable chunks cache, please set -blocks-storage.bucket-store.chunks-cache.backend. Chunks can be stored into Memcached or Redis cache. Memcached client can be configured via flags with -blocks-storage.bucket-store.chunks-cache.memcached.* prefix. Redis client can be configured via flags with -blocks-storage.bucket-store.chunks-cache.redis.* prefix.\nThere are additional low-level options for configuring chunks cache. Please refer to other flags with -blocks-storage.bucket-store.chunks-cache.* prefix.\nMetadata cache Store-gateway and querier can use memcached or redis for caching bucket metadata:\nList of tenants List of blocks per tenant Block’s meta.json content Block’s deletion-mark.json existence and content Tenant’s bucket-index.json.gz content Using the metadata cache can significantly reduce the number of API calls to object storage and protects from linearly scale the number of these API calls with the number of querier and store-gateway instances (because the bucket is periodically scanned and synched by each querier and store-gateway).\nTo enable metadata cache, please set -blocks-storage.bucket-store.metadata-cache.backend. memcached and redis backend are supported currently. Memcached client has additional configuration available via flags with -blocks-storage.bucket-store.metadata-cache.memcached.* prefix. Redis client has additional configuration available via flags with -blocks-storage.bucket-store.metadata-cache.redis.* prefix.\nAdditional options for configuring metadata cache have -blocks-storage.bucket-store.metadata-cache.* prefix. By configuring TTL to zero or negative value, caching of given item type is disabled.\nThe same cache backend deployment should be shared between store-gateways and queriers.\nStore-gateway HTTP endpoints GET /store-gateway/ring\nDisplays the status of the store-gateways ring, including the tokens owned by each store-gateway and an option to remove (forget) instances from the ring. Store-gateway configuration This section described the store-gateway configuration. For the general Cortex configuration and references to common config blocks, please refer to the configuration documentation.\nstore_gateway_config The store_gateway_config configures the store-gateway service used by the blocks storage.\nstore_gateway: # Shard blocks across multiple store gateway instances. This option needs be # set both on the store-gateway and querier when running in microservices # mode. # CLI flag: -store-gateway.sharding-enabled [sharding_enabled: \u003cboolean\u003e | default = false] # The hash ring configuration. This option is required only if blocks sharding # is enabled. sharding_ring: # The key-value store used to share the hash ring across multiple instances. # This option needs be set both on the store-gateway and querier when # running in microservices mode. kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -store-gateway.sharding-ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -store-gateway.sharding-ring.prefix [prefix: \u003cstring\u003e | default = \"collectors/\"] dynamodb: # Region to access dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -store-gateway.sharding-ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -store-gateway.sharding-ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: # store-gateway.sharding-ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: # store-gateway.sharding-ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -store-gateway.sharding-ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -store-gateway.sharding-ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -store-gateway.sharding-ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -store-gateway.sharding-ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -store-gateway.sharding-ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 15s] # The heartbeat timeout after which store gateways are considered unhealthy # within the ring. 0 = never (timeout disabled). This option needs be set # both on the store-gateway and querier when running in microservices mode. # CLI flag: -store-gateway.sharding-ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # The replication factor to use when sharding blocks. This option needs be # set both on the store-gateway and querier when running in microservices # mode. # CLI flag: -store-gateway.sharding-ring.replication-factor [replication_factor: \u003cint\u003e | default = 3] # File path where tokens are stored. If empty, tokens are not stored at # shutdown and restored at startup. # CLI flag: -store-gateway.sharding-ring.tokens-file-path [tokens_file_path: \u003cstring\u003e | default = \"\"] # True to enable zone-awareness and replicate blocks across different # availability zones. # CLI flag: -store-gateway.sharding-ring.zone-awareness-enabled [zone_awareness_enabled: \u003cboolean\u003e | default = false] # True to keep the store gateway instance in the ring when it shuts down. # The instance will then be auto-forgotten from the ring after # 10*heartbeat_timeout. # CLI flag: -store-gateway.sharding-ring.keep-instance-in-the-ring-on-shutdown [keep_instance_in_the_ring_on_shutdown: \u003cboolean\u003e | default = false] # Minimum time to wait for ring stability at startup. 0 to disable. # CLI flag: -store-gateway.sharding-ring.wait-stability-min-duration [wait_stability_min_duration: \u003cduration\u003e | default = 1m] # Maximum time to wait for ring stability at startup. If the store-gateway # ring keeps changing after this period of time, the store-gateway will # start anyway. # CLI flag: -store-gateway.sharding-ring.wait-stability-max-duration [wait_stability_max_duration: \u003cduration\u003e | default = 5m] # Timeout for waiting on store-gateway to become desired state in the ring. # CLI flag: -store-gateway.sharding-ring.wait-instance-state-timeout [wait_instance_state_timeout: \u003cduration\u003e | default = 10m] # The sleep seconds when store-gateway is shutting down. Need to be close to # or larger than KV Store information propagation delay # CLI flag: -store-gateway.sharding-ring.final-sleep [final_sleep: \u003cduration\u003e | default = 0s] # Name of network interface to read address from. # CLI flag: -store-gateway.sharding-ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # The availability zone where this instance is running. Required if # zone-awareness is enabled. # CLI flag: -store-gateway.sharding-ring.instance-availability-zone [instance_availability_zone: \u003cstring\u003e | default = \"\"] # The sharding strategy to use. Supported values are: default, # shuffle-sharding. # CLI flag: -store-gateway.sharding-strategy [sharding_strategy: \u003cstring\u003e | default = \"default\"] # Comma separated list of tenants whose store metrics this storegateway can # process. If specified, only these tenants will be handled by storegateway, # otherwise this storegateway will be enabled for all the tenants in the # store-gateway cluster. # CLI flag: -store-gateway.enabled-tenants [enabled_tenants: \u003cstring\u003e | default = \"\"] # Comma separated list of tenants whose store metrics this storegateway cannot # process. If specified, a storegateway that would normally pick the specified # tenant(s) for processing will ignore them instead. # CLI flag: -store-gateway.disabled-tenants [disabled_tenants: \u003cstring\u003e | default = \"\"] hedged_request: # If true, hedged requests are applied to object store calls. It can help # with reducing tail latency. # CLI flag: -store-gateway.hedged-request.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of hedged requests allowed for each initial request. A high # number can reduce latency but increase internal calls. # CLI flag: -store-gateway.hedged-request.max-requests [max_requests: \u003cint\u003e | default = 3] # It is used to calculate a latency threshold to trigger hedged requests. # For example, additional requests are triggered when the initial request # response time exceeds the 90th percentile. # CLI flag: -store-gateway.hedged-request.quantile [quantile: \u003cfloat\u003e | default = 0.9] blocks_storage_config The blocks_storage_config configures the blocks storage.\nblocks_storage: # Backend storage to use. Supported backends are: s3, gcs, azure, swift, # filesystem. # CLI flag: -blocks-storage.backend [backend: \u003cstring\u003e | default = \"s3\"] s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an # S3-compatible service in hostname:port format. # CLI flag: -blocks-storage.s3.endpoint [endpoint: \u003cstring\u003e | default = \"\"] # S3 region. If unset, the client will issue a S3 GetBucketLocation API call # to autodetect it. # CLI flag: -blocks-storage.s3.region [region: \u003cstring\u003e | default = \"\"] # S3 bucket name # CLI flag: -blocks-storage.s3.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # S3 secret access key # CLI flag: -blocks-storage.s3.secret-access-key [secret_access_key: \u003cstring\u003e | default = \"\"] # S3 access key ID # CLI flag: -blocks-storage.s3.access-key-id [access_key_id: \u003cstring\u003e | default = \"\"] # If enabled, use http:// for the S3 endpoint instead of https://. This # could be useful in local dev/test environments while using an # S3-compatible backend storage, like Minio. # CLI flag: -blocks-storage.s3.insecure [insecure: \u003cboolean\u003e | default = false] # The signature version to use for authenticating against S3. Supported # values are: v4, v2. # CLI flag: -blocks-storage.s3.signature-version [signature_version: \u003cstring\u003e | default = \"v4\"] # The s3 bucket lookup style. Supported values are: auto, virtual-hosted, # path. # CLI flag: -blocks-storage.s3.bucket-lookup-type [bucket_lookup_type: \u003cstring\u003e | default = \"auto\"] # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum # algorithm to verify the provided digest. If false, use CRC32C algorithm # instead. # CLI flag: -blocks-storage.s3.send-content-md5 [send_content_md5: \u003cboolean\u003e | default = true] # The list api version. Supported values are: v1, v2, and ''. # CLI flag: -blocks-storage.s3.list-objects-version [list_objects_version: \u003cstring\u003e | default = \"\"] # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: blocks-storage [sse: \u003cs3_sse_config\u003e] http: # The time an idle connection will remain idle before closing. # CLI flag: -blocks-storage.s3.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -blocks-storage.s3.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -blocks-storage.s3.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -blocks-storage.s3.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully # writing the request headers if the request has an Expect header. 0 to # send the request body immediately. # CLI flag: -blocks-storage.s3.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 # means no limit. # CLI flag: -blocks-storage.s3.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, # a built-in default value is used. # CLI flag: -blocks-storage.s3.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -blocks-storage.s3.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] gcs: # GCS bucket name # CLI flag: -blocks-storage.gcs.bucket-name [bucket_name: \u003cstring\u003e | default = \"\"] # JSON representing either a Google Developers Console # client_credentials.json file or a Google Developers service account key # file. If empty, fallback to Google default logic. # CLI flag: -blocks-storage.gcs.service-account [service_account: \u003cstring\u003e | default = \"\"] azure: # Azure storage account name # CLI flag: -blocks-storage.azure.account-name [account_name: \u003cstring\u003e | default = \"\"] # Azure storage account key # CLI flag: -blocks-storage.azure.account-key [account_key: \u003cstring\u003e | default = \"\"] # The values of `account-name` and `endpoint-suffix` values will not be # ignored if `connection-string` is set. Use this method over `account-key` # if you need to authenticate via a SAS token or if you use the Azurite # emulator. # CLI flag: -blocks-storage.azure.connection-string [connection_string: \u003cstring\u003e | default = \"\"] # Azure storage container name # CLI flag: -blocks-storage.azure.container-name [container_name: \u003cstring\u003e | default = \"\"] # Azure storage endpoint suffix without schema. The account name will be # prefixed to this value to create the FQDN # CLI flag: -blocks-storage.azure.endpoint-suffix [endpoint_suffix: \u003cstring\u003e | default = \"\"] # Number of retries for recoverable errors # CLI flag: -blocks-storage.azure.max-retries [max_retries: \u003cint\u003e | default = 20] # Deprecated: Azure storage MSI resource. It will be set automatically by # Azure SDK. # CLI flag: -blocks-storage.azure.msi-resource [msi_resource: \u003cstring\u003e | default = \"\"] # Azure storage MSI resource managed identity client Id. If not supplied # default Azure credential will be used. Set it to empty if you need to # authenticate via Azure Workload Identity. # CLI flag: -blocks-storage.azure.user-assigned-id [user_assigned_id: \u003cstring\u003e | default = \"\"] http: # The time an idle connection will remain idle before closing. # CLI flag: -blocks-storage.azure.http.idle-conn-timeout [idle_conn_timeout: \u003cduration\u003e | default = 1m30s] # The amount of time the client will wait for a servers response headers. # CLI flag: -blocks-storage.azure.http.response-header-timeout [response_header_timeout: \u003cduration\u003e | default = 2m] # If the client connects via HTTPS and this option is enabled, the client # will accept any certificate and hostname. # CLI flag: -blocks-storage.azure.http.insecure-skip-verify [insecure_skip_verify: \u003cboolean\u003e | default = false] # Maximum time to wait for a TLS handshake. 0 means no limit. # CLI flag: -blocks-storage.azure.tls-handshake-timeout [tls_handshake_timeout: \u003cduration\u003e | default = 10s] # The time to wait for a server's first response headers after fully # writing the request headers if the request has an Expect header. 0 to # send the request body immediately. # CLI flag: -blocks-storage.azure.expect-continue-timeout [expect_continue_timeout: \u003cduration\u003e | default = 1s] # Maximum number of idle (keep-alive) connections across all hosts. 0 # means no limit. # CLI flag: -blocks-storage.azure.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 100] # Maximum number of idle (keep-alive) connections to keep per-host. If 0, # a built-in default value is used. # CLI flag: -blocks-storage.azure.max-idle-connections-per-host [max_idle_connections_per_host: \u003cint\u003e | default = 100] # Maximum number of connections per host. 0 means no limit. # CLI flag: -blocks-storage.azure.max-connections-per-host [max_connections_per_host: \u003cint\u003e | default = 0] swift: # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -blocks-storage.swift.auth-version [auth_version: \u003cint\u003e | default = 0] # OpenStack Swift authentication URL # CLI flag: -blocks-storage.swift.auth-url [auth_url: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential ID. # CLI flag: -blocks-storage.swift.application-credential-id [application_credential_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential name. # CLI flag: -blocks-storage.swift.application-credential-name [application_credential_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift application credential secret. # CLI flag: -blocks-storage.swift.application-credential-secret [application_credential_secret: \u003cstring\u003e | default = \"\"] # OpenStack Swift username. # CLI flag: -blocks-storage.swift.username [username: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -blocks-storage.swift.user-domain-name [user_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -blocks-storage.swift.user-domain-id [user_domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user ID. # CLI flag: -blocks-storage.swift.user-id [user_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift API key. # CLI flag: -blocks-storage.swift.password [password: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain ID. # CLI flag: -blocks-storage.swift.domain-id [domain_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift user's domain name. # CLI flag: -blocks-storage.swift.domain-name [domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift project ID (v2,v3 auth only). # CLI flag: -blocks-storage.swift.project-id [project_id: \u003cstring\u003e | default = \"\"] # OpenStack Swift project name (v2,v3 auth only). # CLI flag: -blocks-storage.swift.project-name [project_name: \u003cstring\u003e | default = \"\"] # ID of the OpenStack Swift project's domain (v3 auth only), only needed if # it differs the from user domain. # CLI flag: -blocks-storage.swift.project-domain-id [project_domain_id: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift project's domain (v3 auth only), only needed # if it differs from the user domain. # CLI flag: -blocks-storage.swift.project-domain-name [project_domain_name: \u003cstring\u003e | default = \"\"] # OpenStack Swift Region to use (v2,v3 auth only). # CLI flag: -blocks-storage.swift.region-name [region_name: \u003cstring\u003e | default = \"\"] # Name of the OpenStack Swift container to put chunks in. # CLI flag: -blocks-storage.swift.container-name [container_name: \u003cstring\u003e | default = \"\"] # Max retries on requests error. # CLI flag: -blocks-storage.swift.max-retries [max_retries: \u003cint\u003e | default = 3] # Time after which a connection attempt is aborted. # CLI flag: -blocks-storage.swift.connect-timeout [connect_timeout: \u003cduration\u003e | default = 10s] # Time after which an idle request is aborted. The timeout watchdog is reset # each time some data is received, so the timeout triggers after X time no # data is received on a request. # CLI flag: -blocks-storage.swift.request-timeout [request_timeout: \u003cduration\u003e | default = 5s] filesystem: # Local filesystem storage directory. # CLI flag: -blocks-storage.filesystem.dir [dir: \u003cstring\u003e | default = \"\"] # This configures how the querier and store-gateway discover and synchronize # blocks stored in the bucket. bucket_store: # Directory to store synchronized TSDB index headers. # CLI flag: -blocks-storage.bucket-store.sync-dir [sync_dir: \u003cstring\u003e | default = \"tsdb-sync\"] # How frequently to scan the bucket, or to refresh the bucket index (if # enabled), in order to look for changes (new blocks shipped by ingesters # and blocks deleted by retention or compaction). # CLI flag: -blocks-storage.bucket-store.sync-interval [sync_interval: \u003cduration\u003e | default = 15m] # Max number of concurrent queries to execute against the long-term storage. # The limit is shared across all tenants. # CLI flag: -blocks-storage.bucket-store.max-concurrent [max_concurrent: \u003cint\u003e | default = 100] # Max number of inflight queries to execute against the long-term storage. # The limit is shared across all tenants. 0 to disable. # CLI flag: -blocks-storage.bucket-store.max-inflight-requests [max_inflight_requests: \u003cint\u003e | default = 0] # Maximum number of concurrent tenants syncing blocks. # CLI flag: -blocks-storage.bucket-store.tenant-sync-concurrency [tenant_sync_concurrency: \u003cint\u003e | default = 10] # Maximum number of concurrent blocks syncing per tenant. # CLI flag: -blocks-storage.bucket-store.block-sync-concurrency [block_sync_concurrency: \u003cint\u003e | default = 20] # Number of Go routines to use when syncing block meta files from object # storage per tenant. # CLI flag: -blocks-storage.bucket-store.meta-sync-concurrency [meta_sync_concurrency: \u003cint\u003e | default = 20] # Minimum age of a block before it's being read. Set it to safe value (e.g # 30m) if your object storage is eventually consistent. GCS and S3 are # (roughly) strongly consistent. # CLI flag: -blocks-storage.bucket-store.consistency-delay [consistency_delay: \u003cduration\u003e | default = 0s] index_cache: # The index cache backend type. Multiple cache backend can be provided as # a comma-separated ordered list to enable the implementation of a cache # hierarchy. Supported values: inmemory, memcached, redis. # CLI flag: -blocks-storage.bucket-store.index-cache.backend [backend: \u003cstring\u003e | default = \"inmemory\"] inmemory: # Maximum size in bytes of in-memory index cache used to speed up blocks # index lookups (shared between all tenants). # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes [max_size_bytes: \u003cint\u003e | default = 1073741824] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items [enabled_items: \u003clist of string\u003e | default = []] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV # query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup # made after that). # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. # If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should # run. If more keys are specified, internally keys are split into # multiple batches and fetched concurrently, honoring the max # concurrency. If set to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items [enabled_items: \u003clist of string\u003e | default = []] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.index-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching # is when data is stored in memory instead of fetching data each time. # See https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # Selectively cache index item types. Supported values are Postings, # ExpandedPostings and Series # CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items [enabled_items: \u003clist of string\u003e | default = []] multilevel: # The maximum number of concurrent asynchronous operations can occur # when backfilling cache items. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of items to backfill per asynchronous operation. # CLI flag: -blocks-storage.bucket-store.index-cache.multilevel.max-backfill-items [max_backfill_items: \u003cint\u003e | default = 10000] chunks_cache: # The chunks cache backend type. Single or Multiple cache backend can be # provided. Supported values in single cache: memcached, redis, inmemory, # and '' (disable). Supported values in multi level cache: a # comma-separated list of (inmemory, memcached, redis) # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend [backend: \u003cstring\u003e | default = \"\"] inmemory: # Maximum size in bytes of in-memory chunk cache used to speed up chunk # lookups (shared between all tenants). # CLI flag: -blocks-storage.bucket-store.chunks-cache.inmemory.max-size-bytes [max_size_bytes: \u003cint\u003e | default = 1073741824] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV # query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup # made after that). # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. # If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should # run. If more keys are specified, internally keys are split into # multiple batches and fetched concurrently, honoring the max # concurrency. If set to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching # is when data is stored in memory instead of fetching data each time. # See https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] multilevel: # The maximum number of concurrent asynchronous operations can occur # when backfilling cache items. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed when # backfilling cache items. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of items to backfill per asynchronous operation. # CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-backfill-items [max_backfill_items: \u003cint\u003e | default = 10000] # Size of each subrange that bucket object is split into for better # caching. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-size [subrange_size: \u003cint\u003e | default = 16000] # Maximum number of sub-GetRange requests that a single GetRange request # can be split into when fetching chunks. Zero or negative value = # unlimited number of sub-requests. # CLI flag: -blocks-storage.bucket-store.chunks-cache.max-get-range-requests [max_get_range_requests: \u003cint\u003e | default = 3] # TTL for caching object attributes for chunks. # CLI flag: -blocks-storage.bucket-store.chunks-cache.attributes-ttl [attributes_ttl: \u003cduration\u003e | default = 168h] # TTL for caching individual chunks subranges. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-ttl [subrange_ttl: \u003cduration\u003e | default = 24h] metadata_cache: # Backend for metadata cache, if not empty. Supported values: memcached, # redis, and '' (disable). # CLI flag: -blocks-storage.bucket-store.metadata-cache.backend [backend: \u003cstring\u003e | default = \"\"] memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV # query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup # made after that). # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.addresses [addresses: \u003cstring\u003e | default = \"\"] # The socket read/write timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.timeout [timeout: \u003cduration\u003e | default = 100ms] # The maximum number of idle connections that will be maintained per # address. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections [max_idle_connections: \u003cint\u003e | default = 16] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # The maximum number of concurrent connections running get operations. # If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum number of keys a single underlying get operation should # run. If more keys are specified, internally keys are split into # multiple batches and fetched concurrently, honoring the max # concurrency. If set to 0, the max batch size is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-batch-size [max_get_multi_batch_size: \u003cint\u003e | default = 0] # The maximum size of an item stored in memcached. Bigger items are not # stored. If set to 0, no maximum size is enforced. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size [max_item_size: \u003cint\u003e | default = 1048576] # Use memcached auto-discovery mechanism provided by some cloud provider # like GCP and AWS # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.auto-discovery [auto_discovery: \u003cboolean\u003e | default = false] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] redis: # Comma separated list of redis addresses. Supported prefixes are: dns+ # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after # that). # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.addresses [addresses: \u003cstring\u003e | default = \"\"] # Redis username. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.username [username: \u003cstring\u003e | default = \"\"] # Redis password. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.password [password: \u003cstring\u003e | default = \"\"] # Database to be selected after connecting to the server. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.db [db: \u003cint\u003e | default = 0] # Specifies the master's name. Must be not empty for Redis Sentinel. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.master-name [master_name: \u003cstring\u003e | default = \"\"] # The maximum number of concurrent GetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-get-multi-concurrency [max_get_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for mget. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.get-multi-batch-size [get_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent SetMulti() operations. If set to 0, # concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-set-multi-concurrency [max_set_multi_concurrency: \u003cint\u003e | default = 100] # The maximum size per batch for pipeline set. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-multi-batch-size [set_multi_batch_size: \u003cint\u003e | default = 100] # The maximum number of concurrent asynchronous operations can occur. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-async-concurrency [max_async_concurrency: \u003cint\u003e | default = 3] # The maximum number of enqueued asynchronous operations allowed. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-async-buffer-size [max_async_buffer_size: \u003cint\u003e | default = 10000] # Client dial timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.dial-timeout [dial_timeout: \u003cduration\u003e | default = 5s] # Client read timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.read-timeout [read_timeout: \u003cduration\u003e | default = 3s] # Client write timeout. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.write-timeout [write_timeout: \u003cduration\u003e | default = 3s] # Whether to enable tls for redis connection. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.tls-enabled [tls_enabled: \u003cboolean\u003e | default = false] # Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-cert-path [tls_cert_path: \u003cstring\u003e | default = \"\"] # Path to the key file for the client certificate. Also requires the # client certificate to be configured. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-key-path [tls_key_path: \u003cstring\u003e | default = \"\"] # Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-ca-path [tls_ca_path: \u003cstring\u003e | default = \"\"] # Override the expected name on the server certificate. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-server-name [tls_server_name: \u003cstring\u003e | default = \"\"] # Skip validating server certificate. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-insecure-skip-verify [tls_insecure_skip_verify: \u003cboolean\u003e | default = false] # If not zero then client-side caching is enabled. Client-side caching # is when data is stored in memory instead of fetching data each time. # See https://redis.io/docs/manual/client-side-caching/ for more info. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.cache-size [cache_size: \u003cint\u003e | default = 0] set_async_circuit_breaker_config: # If true, enable circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.enabled [enabled: \u003cboolean\u003e | default = false] # Maximum number of requests allowed to pass through when the circuit # breaker is half-open. If set to 0, by default it allows 1 request. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.half-open-max-requests [half_open_max_requests: \u003cint\u003e | default = 10] # Period of the open state after which the state of the circuit # breaker becomes half-open. If set to 0, by default open duration is # 60 seconds. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.open-duration [open_duration: \u003cduration\u003e | default = 5s] # Minimal requests to trigger the circuit breaker. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.min-requests [min_requests: \u003cint\u003e | default = 50] # Consecutive failures to determine if the circuit breaker should # open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.consecutive-failures [consecutive_failures: \u003cint\u003e | default = 5] # Failure percentage to determine if the circuit breaker should open. # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: \u003cfloat\u003e | default = 0.05] # How long to cache list of tenants in the bucket. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl [tenants_list_ttl: \u003cduration\u003e | default = 15m] # How long to cache list of blocks for each tenant. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenant-blocks-list-ttl [tenant_blocks_list_ttl: \u003cduration\u003e | default = 5m] # How long to cache list of chunks for a block. # CLI flag: -blocks-storage.bucket-store.metadata-cache.chunks-list-ttl [chunks_list_ttl: \u003cduration\u003e | default = 24h] # How long to cache information that block metafile exists. Also used for # user deletion mark file. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-exists-ttl [metafile_exists_ttl: \u003cduration\u003e | default = 2h] # How long to cache information that block metafile doesn't exist. Also # used for user deletion mark file. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-doesnt-exist-ttl [metafile_doesnt_exist_ttl: \u003cduration\u003e | default = 5m] # How long to cache content of the metafile. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-content-ttl [metafile_content_ttl: \u003cduration\u003e | default = 24h] # Maximum size of metafile content to cache in bytes. Caching will be # skipped if the content exceeds this size. This is useful to avoid # network round trip for large content if the configured caching backend # has an hard limit on cached items size (in this case, you should set # this limit to the same limit in the caching backend). # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-max-size-bytes [metafile_max_size_bytes: \u003cint\u003e | default = 1048576] # How long to cache attributes of the block metafile. # CLI flag: -blocks-storage.bucket-store.metadata-cache.metafile-attributes-ttl [metafile_attributes_ttl: \u003cduration\u003e | default = 168h] # How long to cache attributes of the block index. # CLI flag: -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl [block_index_attributes_ttl: \u003cduration\u003e | default = 168h] # How long to cache content of the bucket index. # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl [bucket_index_content_ttl: \u003cduration\u003e | default = 5m] # Maximum size of bucket index content to cache in bytes. Caching will be # skipped if the content exceeds this size. This is useful to avoid # network round trip for large content if the configured caching backend # has an hard limit on cached items size (in this case, you should set # this limit to the same limit in the caching backend). # CLI flag: -blocks-storage.bucket-store.metadata-cache.bucket-index-max-size-bytes [bucket_index_max_size_bytes: \u003cint\u003e | default = 1048576] # Duration after which the blocks marked for deletion will be filtered out # while fetching blocks. The idea of ignore-deletion-marks-delay is to # ignore blocks that are marked for deletion with some delay. This ensures # store can still serve blocks that are meant to be deleted but do not have # a replacement yet. Default is 6h, half of the default value for # -compactor.deletion-delay. # CLI flag: -blocks-storage.bucket-store.ignore-deletion-marks-delay [ignore_deletion_mark_delay: \u003cduration\u003e | default = 6h] # The blocks created since `now() - ignore_blocks_within` will not be # synced. This should be used together with `-querier.query-store-after` to # filter out the blocks that are too new to be queried. A reasonable value # for this flag would be `-querier.query-store-after - # blocks-storage.bucket-store.bucket-index.max-stale-period` to give some # buffer. 0 to disable. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within [ignore_blocks_within: \u003cduration\u003e | default = 0s] bucket_index: # True to enable querier and store-gateway to discover blocks in the # storage via bucket index instead of bucket scanning. # CLI flag: -blocks-storage.bucket-store.bucket-index.enabled [enabled: \u003cboolean\u003e | default = false] # How frequently a bucket index, which previously failed to load, should # be tried to load again. This option is used only by querier. # CLI flag: -blocks-storage.bucket-store.bucket-index.update-on-error-interval [update_on_error_interval: \u003cduration\u003e | default = 1m] # How long a unused bucket index should be cached. Once this timeout # expires, the unused bucket index is removed from the in-memory cache. # This option is used only by querier. # CLI flag: -blocks-storage.bucket-store.bucket-index.idle-timeout [idle_timeout: \u003cduration\u003e | default = 1h] # The maximum allowed age of a bucket index (last updated) before queries # start failing because the bucket index is too old. The bucket index is # periodically updated by the compactor, while this check is enforced in # the querier (at query time). # CLI flag: -blocks-storage.bucket-store.bucket-index.max-stale-period [max_stale_period: \u003cduration\u003e | default = 1h] # One of concurrent, recursive, bucket_index. When set to concurrent, stores # will concurrently issue one call per directory to discover active blocks # in the bucket. The recursive strategy iterates through all objects in the # bucket, recursively traversing into each directory. This avoids N+1 calls # at the expense of having slower bucket iterations. bucket_index strategy # can be used in Compactor only and utilizes the existing bucket index to # fetch block IDs to sync. This avoids iterating the bucket but can be # impacted by delays of cleaner creating bucket index. # CLI flag: -blocks-storage.bucket-store.block-discovery-strategy [block_discovery_strategy: \u003cstring\u003e | default = \"concurrent\"] # Max size - in bytes - of a chunks pool, used to reduce memory allocations. # The pool is shared across all tenants. 0 to disable the limit. # CLI flag: -blocks-storage.bucket-store.max-chunk-pool-bytes [max_chunk_pool_bytes: \u003cint\u003e | default = 2147483648] # If enabled, store-gateway will lazily memory-map an index-header only once # required by a query. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-enabled [index_header_lazy_loading_enabled: \u003cboolean\u003e | default = false] # If index-header lazy loading is enabled and this setting is \u003e 0, the # store-gateway will release memory-mapped index-headers after 'idle # timeout' inactivity. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout [index_header_lazy_loading_idle_timeout: \u003cduration\u003e | default = 20m] # If true, Store Gateway will estimate postings size and try to lazily # expand postings if it downloads less data than expanding all postings. # CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled [lazy_expanded_postings_enabled: \u003cboolean\u003e | default = false] # Mark posting group as lazy if it fetches more keys than R * max series the # query should fetch. With R set to 100, a posting group which fetches 100K # keys will be marked as lazy if the current query only fetches 1000 series. # This config is only valid if lazy expanded posting is enabled. 0 disables # the limit. # CLI flag: -blocks-storage.bucket-store.lazy-expanded-posting-group-max-key-series-ratio [lazy_expanded_posting_group_max_key_series_ratio: \u003cfloat\u003e | default = 100] # Controls how many series to fetch per batch in Store Gateway. Default # value is 10000. # CLI flag: -blocks-storage.bucket-store.series-batch-size [series_batch_size: \u003cint\u003e | default = 10000] token_bucket_bytes_limiter: # Token bucket bytes limiter mode. Supported values are: disabled, dryrun, # enabled # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.mode [mode: \u003cstring\u003e | default = \"disabled\"] # Instance token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.instance-token-bucket-size [instance_token_bucket_size: \u003cint\u003e | default = 859832320] # User token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.user-token-bucket-size [user_token_bucket_size: \u003cint\u003e | default = 644874240] # Request token bucket size # CLI flag: -blocks-storage.bucket-store.token-bucket-bytes-limiter.request-token-bucket-size [request_token_bucket_size: \u003cint\u003e | default = 4194304] tsdb: # Local directory to store TSDBs in the ingesters. # CLI flag: -blocks-storage.tsdb.dir [dir: \u003cstring\u003e | default = \"tsdb\"] # TSDB blocks range period. # CLI flag: -blocks-storage.tsdb.block-ranges-period [block_ranges_period: \u003clist of duration\u003e | default = 2h0m0s] # TSDB blocks retention in the ingester before a block is removed. This # should be larger than the block_ranges_period and large enough to give # store-gateways and queriers enough time to discover newly uploaded blocks. # CLI flag: -blocks-storage.tsdb.retention-period [retention_period: \u003cduration\u003e | default = 6h] # How frequently the TSDB blocks are scanned and new ones are shipped to the # storage. 0 means shipping is disabled. # CLI flag: -blocks-storage.tsdb.ship-interval [ship_interval: \u003cduration\u003e | default = 1m] # Maximum number of tenants concurrently shipping blocks to the storage. # CLI flag: -blocks-storage.tsdb.ship-concurrency [ship_concurrency: \u003cint\u003e | default = 10] # How frequently does Cortex try to compact TSDB head. Block is only created # if data covers smallest block range. Must be greater than 0 and max 30 # minutes. Note that up to 50% jitter is added to the value for the first # compaction to avoid ingesters compacting concurrently. # CLI flag: -blocks-storage.tsdb.head-compaction-interval [head_compaction_interval: \u003cduration\u003e | default = 1m] # Maximum number of tenants concurrently compacting TSDB head into a new # block # CLI flag: -blocks-storage.tsdb.head-compaction-concurrency [head_compaction_concurrency: \u003cint\u003e | default = 5] # If TSDB head is idle for this duration, it is compacted. Note that up to # 25% jitter is added to the value to avoid ingesters compacting # concurrently. 0 means disabled. # CLI flag: -blocks-storage.tsdb.head-compaction-idle-timeout [head_compaction_idle_timeout: \u003cduration\u003e | default = 1h] # The write buffer size used by the head chunks mapper. Lower values reduce # memory utilisation on clusters with a large number of tenants at the cost # of increased disk I/O operations. # CLI flag: -blocks-storage.tsdb.head-chunks-write-buffer-size-bytes [head_chunks_write_buffer_size_bytes: \u003cint\u003e | default = 4194304] # The number of shards of series to use in TSDB (must be a power of 2). # Reducing this will decrease memory footprint, but can negatively impact # performance. # CLI flag: -blocks-storage.tsdb.stripe-size [stripe_size: \u003cint\u003e | default = 16384] # Deprecated (use blocks-storage.tsdb.wal-compression-type instead): True to # enable TSDB WAL compression. # CLI flag: -blocks-storage.tsdb.wal-compression-enabled [wal_compression_enabled: \u003cboolean\u003e | default = false] # TSDB WAL type. Supported values are: 'snappy', 'zstd' and '' (disable # compression) # CLI flag: -blocks-storage.tsdb.wal-compression-type [wal_compression_type: \u003cstring\u003e | default = \"\"] # TSDB WAL segments files max size (bytes). # CLI flag: -blocks-storage.tsdb.wal-segment-size-bytes [wal_segment_size_bytes: \u003cint\u003e | default = 134217728] # True to flush blocks to storage on shutdown. If false, incomplete blocks # will be reused after restart. # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown [flush_blocks_on_shutdown: \u003cboolean\u003e | default = false] # If TSDB has not received any data for this duration, and all blocks from # TSDB have been shipped, TSDB is closed and deleted from local disk. If set # to positive value, this value should be equal or higher than # -querier.query-ingesters-within flag to make sure that TSDB is not closed # prematurely, which could cause partial query results. 0 or negative value # disables closing of idle TSDB. # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout [close_idle_tsdb_timeout: \u003cduration\u003e | default = 0s] # The size of the in-memory queue used before flushing chunks to the disk. # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: \u003cint\u003e | default = 0] # limit the number of concurrently opening TSDB's on startup # CLI flag: -blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup [max_tsdb_opening_concurrency_on_startup: \u003cint\u003e | default = 10] # Deprecated, use maxExemplars in limits instead. If the MaxExemplars value # in limits is set to zero, cortex will fallback on this value. This setting # enables support for exemplars in TSDB and sets the maximum number that # will be stored. 0 or less means disabled. # CLI flag: -blocks-storage.tsdb.max-exemplars [max_exemplars: \u003cint\u003e | default = 0] # True to enable snapshotting of in-memory TSDB data on disk when shutting # down. # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown [memory_snapshot_on_shutdown: \u003cboolean\u003e | default = false] # [EXPERIMENTAL] Configures the maximum number of samples per chunk that can # be out-of-order. # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max [out_of_order_cap_max: \u003cint\u003e | default = 32] # [EXPERIMENTAL] True to enable native histogram. # CLI flag: -blocks-storage.tsdb.enable-native-histograms [enable_native_histograms: \u003cboolean\u003e | default = false] # [EXPERIMENTAL] If enabled, ingesters will cache expanded postings when # querying blocks. Caching can be configured separately for the head and # compacted blocks. expanded_postings_cache: # If enabled, ingesters will cache expanded postings for the head block. # Only queries with with an equal matcher for metric __name__ are cached. head: # Whether the postings cache is enabled or not # CLI flag: -blocks-storage.expanded_postings_cache.head.enabled [enabled: \u003cboolean\u003e | default = false] # Max bytes for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.head.max-bytes [max_bytes: \u003cint\u003e | default = 10485760] # TTL for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.head.ttl [ttl: \u003cduration\u003e | default = 10m] # If enabled, ingesters will cache expanded postings for the compacted # blocks. The cache is shared between all blocks. blocks: # Whether the postings cache is enabled or not # CLI flag: -blocks-storage.expanded_postings_cache.block.enabled [enabled: \u003cboolean\u003e | default = false] # Max bytes for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.block.max-bytes [max_bytes: \u003cint\u003e | default = 10485760] # TTL for postings cache # CLI flag: -blocks-storage.expanded_postings_cache.block.ttl [ttl: \u003cduration\u003e | default = 10m] ","categories":"","description":"","excerpt":" The store-gateway is the Cortex service responsible to query series …","ref":"/docs/blocks-storage/store-gateway/","tags":"","title":"Store-gateway"},{"body":" The compactor is an service which is responsible to:\nCompact multiple blocks of a given tenant into a single optimized larger block. This helps to reduce storage costs (deduplication, index size reduction), and increase query speed (querying fewer blocks is faster). Keep the per-tenant bucket index updated. The bucket index is used by queriers, store-gateways and rulers to discover new blocks in the storage. The compactor is stateless.\nHow compaction works The blocks compaction has two main benefits:\nVertically compact blocks uploaded by all ingesters for the same time range Horizontally compact blocks with small time ranges into a single larger block The vertical compaction merges all the blocks of a tenant uploaded by ingesters for the same time range (2 hours ranges by default) into a single block, while de-duplicating samples that are originally written to N blocks as a result of replication. This step reduces number of blocks for single 2 hours time range from #(number of ingesters) to 1 per tenant.\nThe horizontal compaction triggers after the vertical compaction and compacts several blocks with adjacent 2-hour range periods into a single larger block. Even though the total size of block chunks doesn’t change after this compaction, it may still significantly reduce the size of the index and the index-header kept in memory by store-gateways.\nCompactor sharding The compactor optionally supports sharding.\nWhen sharding is enabled, multiple compactor instances can coordinate to split the workload and shard blocks by tenant. All the blocks of a tenant are processed by a single compactor instance at any given time, but compaction for different tenants may simultaneously run on different compactor instances.\nWhenever the pool of compactors increase or decrease (ie. following up a scale up/down), tenants are resharded across the available compactor instances without any manual intervention.\nThe compactor sharding is based on the Cortex hash ring. At startup, a compactor generates random tokens and registers itself to the ring. While running, it periodically scans the storage bucket (every -compactor.compaction-interval) to discover the list of tenants in the storage and compacts blocks for each tenant whose hash matches the token ranges assigned to the instance itself within the ring.\nThis feature can be enabled via -compactor.sharding-enabled=true and requires the backend hash ring to be configured via -compactor.ring.* flags (or their respective YAML config options).\nWaiting for stable ring at startup In the event of a cluster cold start or scale up of 2+ compactor instances at the same time we may end up in a situation where each new compactor instance starts at a slightly different time and thus each one runs the first compaction based on a different state of the ring. This is not a critical condition, but may be inefficient, because multiple compactor replicas may start compacting the same tenant nearly at the same time.\nTo reduce the likelihood this could happen, the compactor waits for a stable ring at startup. A ring is considered stable if no instance is added/removed to the ring for at least -compactor.ring.wait-stability-min-duration. If the ring keep getting changed after -compactor.ring.wait-stability-max-duration, the compactor will stop waiting for a stable ring and will proceed starting up normally.\nTo disable this waiting logic, you can start the compactor with -compactor.ring.wait-stability-min-duration=0.\nSoft and hard blocks deletion When the compactor successfully compacts some source blocks into a larger block, source blocks are deleted from the storage. Blocks deletion is not immediate, but follows a two steps process:\nFirst, a block is marked for deletion (soft delete) Then, once a block is marked for deletion for longer then -compactor.deletion-delay, the block is deleted from the storage (hard delete) The compactor is both responsible to mark blocks for deletion and then hard delete them once the deletion delay expires. The soft deletion is based on a tiny deletion-mark.json file stored within the block location in the bucket which gets looked up both by queriers and store-gateways.\nThis soft deletion mechanism is used to give enough time to queriers and store-gateways to discover the new compacted blocks before the old source blocks are deleted. If source blocks would be immediately hard deleted by the compactor, some queries involving the compacted blocks may fail until the queriers and store-gateways haven’t rescanned the bucket and found both deleted source blocks and the new compacted ones.\nCompactor disk utilization The compactor needs to download source blocks from the bucket to the local disk, and store the compacted block to the local disk before uploading it to the bucket. Depending on the largest tenants in your cluster and the configured -compactor.block-ranges, the compactor may need a lot of disk space.\nAssuming max_compaction_range_blocks_size is the total size of blocks for the largest tenant (you can measure it inspecting the bucket) and the longest -compactor.block-ranges period, the formula to estimate the minimum disk space required is:\nmin_disk_space_required = compactor.compaction-concurrency * max_compaction_range_blocks_size * 2 Alternatively, assuming the largest -compactor.block-ranges is 24h (default), you could consider 150GB of disk space every 10M active series owned by the largest tenant. For example, if your largest tenant has 30M active series and -compactor.compaction-concurrency=1 we would recommend having a disk with at least 450GB available.\nCompactor HTTP endpoints GET /compactor/ring\nDisplays the status of the compactors ring, including the tokens owned by each compactor and an option to remove (forget) instances from the ring. Compactor configuration This section described the compactor configuration. For the general Cortex configuration and references to common config blocks, please refer to the configuration documentation.\ncompactor_config The compactor_config configures the compactor for the blocks storage.\ncompactor: # List of compaction time ranges. # CLI flag: -compactor.block-ranges [block_ranges: \u003clist of duration\u003e | default = 2h0m0s,12h0m0s,24h0m0s] # Number of Go routines to use when syncing block index and chunks files from # the long term storage. # CLI flag: -compactor.block-sync-concurrency [block_sync_concurrency: \u003cint\u003e | default = 20] # Number of Go routines to use when syncing block meta files from the long # term storage. # CLI flag: -compactor.meta-sync-concurrency [meta_sync_concurrency: \u003cint\u003e | default = 20] # Minimum age of fresh (non-compacted) blocks before they are being processed. # Malformed blocks older than the maximum of consistency-delay and 48h0m0s # will be removed. # CLI flag: -compactor.consistency-delay [consistency_delay: \u003cduration\u003e | default = 0s] # Data directory in which to cache blocks and process compactions # CLI flag: -compactor.data-dir [data_dir: \u003cstring\u003e | default = \"./data\"] # The frequency at which the compaction runs # CLI flag: -compactor.compaction-interval [compaction_interval: \u003cduration\u003e | default = 1h] # How many times to retry a failed compaction within a single compaction run. # CLI flag: -compactor.compaction-retries [compaction_retries: \u003cint\u003e | default = 3] # Max number of concurrent compactions running. # CLI flag: -compactor.compaction-concurrency [compaction_concurrency: \u003cint\u003e | default = 1] # How frequently compactor should run blocks cleanup and maintenance, as well # as update the bucket index. # CLI flag: -compactor.cleanup-interval [cleanup_interval: \u003cduration\u003e | default = 15m] # Max number of tenants for which blocks cleanup and maintenance should run # concurrently. # CLI flag: -compactor.cleanup-concurrency [cleanup_concurrency: \u003cint\u003e | default = 20] # Time before a block marked for deletion is deleted from bucket. If not 0, # blocks will be marked for deletion and compactor component will permanently # delete blocks marked for deletion from the bucket. If 0, blocks will be # deleted straight away. Note that deleting blocks immediately can cause query # failures. # CLI flag: -compactor.deletion-delay [deletion_delay: \u003cduration\u003e | default = 12h] # For tenants marked for deletion, this is time between deleting of last # block, and doing final cleanup (marker files, debug files) of the tenant. # CLI flag: -compactor.tenant-cleanup-delay [tenant_cleanup_delay: \u003cduration\u003e | default = 6h] # When enabled, mark blocks containing index with out-of-order chunks for no # compact instead of halting the compaction. # CLI flag: -compactor.skip-blocks-with-out-of-order-chunks-enabled [skip_blocks_with_out_of_order_chunks_enabled: \u003cboolean\u003e | default = false] # Number of goroutines to use when fetching/uploading block files from object # storage. # CLI flag: -compactor.block-files-concurrency [block_files_concurrency: \u003cint\u003e | default = 10] # Number of goroutines to use when fetching blocks from object storage when # compacting. # CLI flag: -compactor.blocks-fetch-concurrency [blocks_fetch_concurrency: \u003cint\u003e | default = 3] # When enabled, at compactor startup the bucket will be scanned and all found # deletion marks inside the block location will be copied to the markers # global location too. This option can (and should) be safely disabled as soon # as the compactor has successfully run at least once. # CLI flag: -compactor.block-deletion-marks-migration-enabled [block_deletion_marks_migration_enabled: \u003cboolean\u003e | default = false] # Comma separated list of tenants that can be compacted. If specified, only # these tenants will be compacted by compactor, otherwise all tenants can be # compacted. Subject to sharding. # CLI flag: -compactor.enabled-tenants [enabled_tenants: \u003cstring\u003e | default = \"\"] # Comma separated list of tenants that cannot be compacted by this compactor. # If specified, and compactor would normally pick given tenant for compaction # (via -compactor.enabled-tenants or sharding), it will be ignored instead. # CLI flag: -compactor.disabled-tenants [disabled_tenants: \u003cstring\u003e | default = \"\"] # Shard tenants across multiple compactor instances. Sharding is required if # you run multiple compactor instances, in order to coordinate compactions and # avoid race conditions leading to the same tenant blocks simultaneously # compacted by different instances. # CLI flag: -compactor.sharding-enabled [sharding_enabled: \u003cboolean\u003e | default = false] # The sharding strategy to use. Supported values are: default, # shuffle-sharding. # CLI flag: -compactor.sharding-strategy [sharding_strategy: \u003cstring\u003e | default = \"default\"] sharding_ring: kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. # CLI flag: -compactor.ring.store [store: \u003cstring\u003e | default = \"consul\"] # The prefix for the keys in the store. Should end with a /. # CLI flag: -compactor.ring.prefix [prefix: \u003cstring\u003e | default = \"collectors/\"] dynamodb: # Region to access dynamodb. # CLI flag: -compactor.ring.dynamodb.region [region: \u003cstring\u003e | default = \"\"] # Table name to use on dynamodb. # CLI flag: -compactor.ring.dynamodb.table-name [table_name: \u003cstring\u003e | default = \"\"] # Time to expire items on dynamodb. # CLI flag: -compactor.ring.dynamodb.ttl-time [ttl: \u003cduration\u003e | default = 0s] # Time to refresh local ring with information on dynamodb. # CLI flag: -compactor.ring.dynamodb.puller-sync-time [puller_sync_time: \u003cduration\u003e | default = 1m] # Maximum number of retries for DDB KV CAS. # CLI flag: -compactor.ring.dynamodb.max-cas-retries [max_cas_retries: \u003cint\u003e | default = 10] # The consul_config configures the consul client. # The CLI flags prefix for this block config is: compactor.ring [consul: \u003cconsul_config\u003e] # The etcd_config configures the etcd client. # The CLI flags prefix for this block config is: compactor.ring [etcd: \u003cetcd_config\u003e] multi: # Primary backend storage used by multi-client. # CLI flag: -compactor.ring.multi.primary [primary: \u003cstring\u003e | default = \"\"] # Secondary backend storage used by multi-client. # CLI flag: -compactor.ring.multi.secondary [secondary: \u003cstring\u003e | default = \"\"] # Mirror writes to secondary store. # CLI flag: -compactor.ring.multi.mirror-enabled [mirror_enabled: \u003cboolean\u003e | default = false] # Timeout for storing value to secondary store. # CLI flag: -compactor.ring.multi.mirror-timeout [mirror_timeout: \u003cduration\u003e | default = 2s] # Period at which to heartbeat to the ring. 0 = disabled. # CLI flag: -compactor.ring.heartbeat-period [heartbeat_period: \u003cduration\u003e | default = 5s] # The heartbeat timeout after which compactors are considered unhealthy # within the ring. 0 = never (timeout disabled). # CLI flag: -compactor.ring.heartbeat-timeout [heartbeat_timeout: \u003cduration\u003e | default = 1m] # Minimum time to wait for ring stability at startup. 0 to disable. # CLI flag: -compactor.ring.wait-stability-min-duration [wait_stability_min_duration: \u003cduration\u003e | default = 1m] # Maximum time to wait for ring stability at startup. If the compactor ring # keeps changing after this period of time, the compactor will start anyway. # CLI flag: -compactor.ring.wait-stability-max-duration [wait_stability_max_duration: \u003cduration\u003e | default = 5m] # Name of network interface to read address from. # CLI flag: -compactor.ring.instance-interface-names [instance_interface_names: \u003clist of string\u003e | default = [eth0 en0]] # File path where tokens are stored. If empty, tokens are not stored at # shutdown and restored at startup. # CLI flag: -compactor.ring.tokens-file-path [tokens_file_path: \u003cstring\u003e | default = \"\"] # Unregister the compactor during shutdown if true. # CLI flag: -compactor.ring.unregister-on-shutdown [unregister_on_shutdown: \u003cboolean\u003e | default = true] # Timeout for waiting on compactor to become ACTIVE in the ring. # CLI flag: -compactor.ring.wait-active-instance-timeout [wait_active_instance_timeout: \u003cduration\u003e | default = 10m] # The compaction strategy to use. Supported values are: default, partitioning. # CLI flag: -compactor.compaction-mode [compaction_mode: \u003cstring\u003e | default = \"default\"] # How long block visit marker file should be considered as expired and able to # be picked up by compactor again. # CLI flag: -compactor.block-visit-marker-timeout [block_visit_marker_timeout: \u003cduration\u003e | default = 5m] # How frequently block visit marker file should be updated duration # compaction. # CLI flag: -compactor.block-visit-marker-file-update-interval [block_visit_marker_file_update_interval: \u003cduration\u003e | default = 1m] # How long cleaner visit marker file should be considered as expired and able # to be picked up by cleaner again. The value should be smaller than # -compactor.cleanup-interval # CLI flag: -compactor.cleaner-visit-marker-timeout [cleaner_visit_marker_timeout: \u003cduration\u003e | default = 10m] # How frequently cleaner visit marker file should be updated when cleaning # user. # CLI flag: -compactor.cleaner-visit-marker-file-update-interval [cleaner_visit_marker_file_update_interval: \u003cduration\u003e | default = 5m] # When enabled, index verification will ignore out of order label names. # CLI flag: -compactor.accept-malformed-index [accept_malformed_index: \u003cboolean\u003e | default = false] # When enabled, caching bucket will be used for compactor, except cleaner # service, which serves as the source of truth for block status # CLI flag: -compactor.caching-bucket-enabled [caching_bucket_enabled: \u003cboolean\u003e | default = false] ","categories":"","description":"","excerpt":" The compactor is an service which is responsible to:\nCompact multiple …","ref":"/docs/blocks-storage/compactor/","tags":"","title":"Compactor"},{"body":"Cortex requires a Key-Value (KV) store to store the ring. It can use traditional KV stores like Consul or Etcd, but it can also build its own KV store on top of the memberlist library using a gossip algorithm.\nThis short guide shows how to start Cortex in single-binary mode with a memberlist-based ring. To reduce the number of required dependencies in this guide, it will use blocks storage with no shipping to external stores. The storage engine and external storage configuration are not dependent on the ring configuration.\nSingle-binary, two Cortex instances For simplicity and to get started, we’ll run it as two instances of Cortex on the local computer. We will use prepared configuration files (file 1, file 2), with no external dependencies.\nBuild Cortex first:\n$ go build ./cmd/cortex Run two instances of Cortex, each one with its own dedicated config file:\n$ ./cortex -config.file docs/configuration/single-process-config-blocks-gossip-1.yaml $ ./cortex -config.file docs/configuration/single-process-config-blocks-gossip-2.yaml Download Prometheus and configure it to use our first Cortex instance for remote writes.\nremote_write: - url: http://localhost:9109/api/v1/push After starting Prometheus, it will now start pushing data to Cortex. The Distributor component in Cortex will distribute incoming samples between the two instances.\nTo query that data, you can configure your Grafana instance to use http://localhost:9109/prometheus (first Cortex) as a Prometheus data source.\nHow it works The two instances we started earlier should be able to find each other via memberlist configuration (already present in the config files):\nmemberlist: # defaults to hostname node_name: \"Ingester 1\" bind_port: 7946 join_members: - localhost:7947 abort_if_cluster_join_fails: false This tells memberlist to listen on port 7946, and connect to localhost:7947, which is the second instance. Port numbers are reversed in the second configuration file. We also need to configure node_name and also ingester ID (ingester.lifecycler.id field), because it defaults to hostname, but we are running both Cortex instances on the same host.\nTo make sure that both ingesters generate unique tokens, we configure join_after and observe_period to 10 seconds. The first option tells Cortex to wait 10 seconds before joining the ring. This option is normally used to tell Cortex ingester how long to wait for a potential token and data transfer from leaving ingester, but we also use it here to increase the chance of finding other gossip peers. When Cortex joins the ring, it generates tokens and writes them to the ring. If multiple Cortex instances do this at the same time, they can generate conflicting tokens. This can be a problem when using a gossiped ring (instances may simply not see each other yet), so we use observe_period to watch the ring for token conflicts. If conflict is detected, new tokens are generated instead of conflicting tokens, and the observe period is restarted. If no conflict is detected within the observe period, the ingester switches to the ACTIVE state.\nWe are able to observe ring state on http://localhost:9109/ring and http://localhost:9209/ring. The two instances may see slightly different views (eg. different timestamps), but they should converge to a common state soon, with both instances being ACTIVE and ready to receive samples.\nHow to add another instance? To add another Cortex to the small cluster, copy docs/configuration/single-process-config-blocks-gossip-1.yaml to a new file, and make the following modifications. We assume that the third Cortex will run on the same machine again, so we change the node name and ingester ID as well. Here is the annotated diff:\n... server: + # These ports need to be unique. - http_listen_port: 9109 - grpc_listen_port: 9195 + http_listen_port: 9309 + grpc_listen_port: 9395 ... ingester: lifecycler: # Defaults to hostname, but we run both ingesters in this demonstration on the same machine. - id: \"Ingester 1\" + id: \"Ingester 3\" ... memberlist: # defaults to hostname - node_name: \"Ingester 1\" + node_name: \"Ingester 3\" # bind_port needs to be unique - bind_port: 7946 + bind_port: 7948 ... +# Directory names in the `blocks_storage` \u003e `tsdb` config ending with `...1` to end with `...3`. This is to avoid different instances +# writing in-progress data to the same directories. blocks_storage: tsdb: - dir: /tmp/cortex/tsdb-ing1 + dir: /tmp/cortex/tsdb-ing3 bucket_store: - sync_dir: /tmp/cortex/tsdb-sync-querier1 + sync_dir: /tmp/cortex/tsdb-sync-querier3 ... We don’t need to change or add memberlist.join_members list. This new instance will simply join to the second one (listening on port 7947), and will discover other peers through it. When using Kubernetes, the suggested setup is to have a headless service pointing to all pods that want to be part of the gossip cluster, and then point join_members to this headless service.\nWe also don’t need to change /tmp/cortex/storage directory in the blocks_storage.filesystem.dir field. This is the directory where all ingesters will “upload” finished blocks. This can also be an S3 or GCP storage, but for simplicity, we use the local filesystem in this example.\nAfter these changes, we can start another Cortex instance using the modified configuration file. This instance will join the ring and will start receiving samples after it enters the ACTIVE state.\n","categories":"","description":"","excerpt":"Cortex requires a Key-Value (KV) store to store the ring. It can use …","ref":"/docs/guides/getting-started-with-gossiped-ring/","tags":"","title":"Getting started with gossiped ring"},{"body":"To upgrade the Go’s runtime version:\nUpgrade build image version with golang version as describe here If the minimum support Go’s language version should be upgraded as well:\nUpgrade go version in go.mod ","categories":"","description":"","excerpt":"To upgrade the Go’s runtime version:\nUpgrade build image version with …","ref":"/docs/contributing/how-to-upgrade-golang-version/","tags":"","title":"How to upgrade Golang version"},{"body":"This page shares some tips and things to take in consideration when setting up a production Cortex cluster based on the blocks storage.\nIngester Ensure a high number of max open file descriptors The ingester stores received series into per-tenant TSDB blocks. Both TSDB WAL, head and compacted blocks are composed by a relatively large number of files which gets loaded via mmap. This means that the ingester keeps file descriptors open for TSDB WAL segments, chunk files and compacted blocks which haven’t reached the retention period yet.\nIf your Cortex cluster has many tenants or ingester is running with a long -blocks-storage.tsdb.retention-period, the ingester may hit the file-max ulimit (maximum number of open file descriptions by a process); in such case, we recommend increasing the limit on your system or enabling shuffle sharding.\nThe rule of thumb is that a production system shouldn’t have the file-max ulimit below 65536, but higher values are recommended (eg. 1048576).\nIngester disk space Ingesters create blocks on disk as samples come in, then every 2 hours (configurable) they cut off those blocks and start a new block.\nWe typically configure ingesters to retain these blocks for longer, to allow time to recover if something goes wrong uploading to the long-term store and to reduce work in queriers - more detail here.\nIf you configure ingesters with -blocks-storage.tsdb.retention-period=24h, a rule of thumb for disk space required is to take the number of timeseries after replication and multiply by 30KB.\nFor example, if you have 20M active series replicated 3 ways, this gives approx 1.7TB. Divide by the number of ingesters and allow some margin for growth, e.g. if you have 20 ingesters then 100GB each should work, or 150GB each to be more comfortable.\nQuerier Ensure caching is enabled The querier relies on caching to reduce the number API calls to the storage bucket. Ensure caching is properly configured and properly scaled.\nEnsure bucket index is enabled The bucket index reduces the number of API calls to the storage bucket and, when enabled, the querier is up and running immediately after the startup (no need to run an initial bucket scan). Ensure bucket index is enabled for the querier.\nAvoid querying non compacted blocks When running Cortex blocks storage cluster at scale, querying non compacted blocks may be inefficient for two reasons:\nNon compacted blocks contain duplicated samples (as effect of the ingested samples replication) Overhead introduced querying many small indexes Because of this, we would suggest to avoid querying non compacted blocks. In order to do it, you should:\nRun the compactor Configure queriers -querier.query-store-after large enough to give compactor enough time to compact newly uploaded blocks (see below) Configure queriers -querier.query-ingesters-within equal to -querier.query-store-after plus 5m (5 minutes is just a delta to query the boundary both from ingesters and queriers) Configure ingesters -blocks-storage.tsdb.retention-period at least as -querier.query-ingesters-within Lower -blocks-storage.bucket-store.ignore-deletion-marks-delay to 1h, otherwise non compacted blocks could be queried anyway, even if their compacted replacement is available How to estimate -querier.query-store-after The -querier.query-store-after should be set to a duration large enough to give compactor enough time to compact newly uploaded blocks, and queriers and store-gateways to discover and sync newly compacted blocks.\nThe following diagram shows all the timings involved in the estimation. This diagram should be used only as a template and you’re expected to tweak the assumptions based on real measurements in your Cortex cluster. In this example, the following assumptions have been done:\nAn ingester takes up to 30 minutes to upload a block to the storage The compactor takes up to 3 hours to compact 2h blocks shipped from all ingesters Querier and store-gateways take up to 15 minutes to discover and load a new compacted block Given these assumptions, in the worst case scenario it would take up to 6h and 45m since when a sample has been ingested until that sample has been appended to a block flushed to the storage and that block has been vertically compacted with all other overlapping 2h blocks shipped from ingesters.\nStore-gateway Ensure caching is enabled The store-gateway heavily relies on caching both to speed up the queries and to reduce the number of API calls to the storage bucket. Ensure caching is properly configured and properly scaled.\nEnsure bucket index is enabled The bucket index reduces the number of API calls to the storage bucket and the startup time of the store-gateway. Ensure bucket index is enabled for the store-gateway.\nEnsure a high number of max open file descriptors The store-gateway stores each block’s index-header on the local disk and loads it via mmap. This means that the store-gateway keeps a file descriptor open for each loaded block. If your Cortex cluster has many blocks in the bucket, the store-gateway may hit the file-max ulimit (maximum number of open file descriptions by a process); in such case, we recommend increasing the limit on your system or running more store-gateway instances with blocks sharding enabled.\nThe rule of thumb is that a production system shouldn’t have the file-max ulimit below 65536, but higher values are recommended (eg. 1048576).\nCompactor Ensure the compactor has enough disk space The compactor generally needs a lot of disk space in order to download source blocks from the bucket and store the compacted block before uploading it to the storage. Please refer to Compactor disk utilization for more information about how to do capacity planning.\nCaching Ensure memcached is properly scaled The rule of thumb to ensure memcached is properly scaled is to make sure evictions happen infrequently. When that’s not the case and they affect query performances, the suggestion is to scale out the memcached cluster adding more nodes or increasing the memory limit of existing ones.\nWe also recommend to run a different memcached cluster for each cache type (metadata, index, chunks). It’s not required, but suggested to not worry about the effect of memory pressure on a cache type against others.\nAlertmanager Ensure Alertmanager networking is hardened If the Alertmanager API is enabled, users with access to Cortex can autonomously configure the Alertmanager, including receiver integrations that allow to issue network requests to the configured URL (eg. webhook). If the Alertmanager network is not hardened, Cortex users may have the ability to issue network requests to any network endpoint including services running in the local network accessible by the Alertmanager itself.\nDespite hardening the system is out of the scope of Cortex, Cortex provides a basic built-in firewall to block connections created by Alertmanager receiver integrations:\n-alertmanager.receivers-firewall-block-cidr-networks -alertmanager.receivers-firewall-block-private-addresses These settings can also be overridden on a per-tenant basis via overrides specified in the runtime config.\n","categories":"","description":"","excerpt":"This page shares some tips and things to take in consideration when …","ref":"/docs/blocks-storage/production-tips/","tags":"","title":"Production tips"},{"body":"Requests mirroring (or shadowing) is a technique you can use to mirror requests from a primary Cortex cluster to a secondary one.\nFor example, requests mirroring can be used when you need to setup a testing Cortex cluster receiving the same series ingested by a primary one without having control over Prometheus remote write config (if you do, then configuring two remote write entries in Prometheus would be the preferred option).\nMirroring with Envoy proxy Envoy proxy can be used to mirror HTTP requests to a secondary upstream cluster. From a network path perspective, you should run Envoy in front of both clusters distributors, letting Envoy to proxy requests to the primary Cortex cluster and mirror them to a secondary cluster in background. The performances and availability of the secondary cluster have no impact on the requests to the primary one. The response to the client will always be the one from the primary one. In this sense, the requests from Envoy to the secondary cluster are “fire and forget”.\nExample Envoy config The following Envoy configuration shows an example with two Cortex clusters. Envoy will listen on port 9900 and will proxies all requests to cortex-primary:80, mirroring it to cortex-secondary:80 too.\nadmin: # No access logs. access_log_path: /dev/null address: socket_address: { address: 0.0.0.0, port_value: 9901 } static_resources: listeners: - name: cortex_listener address: socket_address: { address: 0.0.0.0, port_value: 9900 } filter_chains: - filters: - name: envoy.http_connection_manager config: stat_prefix: cortex_ingress route_config: name: all_routes virtual_hosts: - name: all_hosts domains: [\"*\"] routes: - match: { prefix: \"/\" } route: cluster: cortex_primary # Specifies the upstream timeout. This spans between the point at which the entire downstream # request has been processed and when the upstream response has been completely processed. timeout: 15s # Specifies the cluster that requests will be mirrored to. The performances and availability of # the secondary cluster have no impact on the requests to the primary one. The response to the # client will always be the one from the primary one. The requests from Envoy to the secondary # cluster are \"fire and forget\". request_mirror_policies: - cluster: cortex_secondary http_filters: - name: envoy.router clusters: - name: cortex_primary type: STRICT_DNS connect_timeout: 1s hosts: [{ socket_address: { address: cortex-primary, port_value: 80 }}] dns_refresh_rate: 5s - name: cortex_secondary type: STRICT_DNS connect_timeout: 1s hosts: [{ socket_address: { address: cortex-secondary, port_value: 80 }}] dns_refresh_rate: 5s ","categories":"","description":"","excerpt":"Requests mirroring (or shadowing) is a technique you can use to mirror …","ref":"/docs/operations/requests-mirroring-to-secondary-cluster/","tags":"","title":"Requests mirroring to secondary cluster"},{"body":"In order to query series inside blocks from object storage, the store-gateway has to know certain initial info from each block index. In order to achieve so, the store-gateway builds an index-header for each block and stores it on local disk; such index-header is built by downloading specific pieces of original block’s index and storing them on local disk. Index header is then used by store-gateway at query time.\nStore-gateways build the index-header with specific sections of the block’s index downloaded using GET byte range requests. Since downloading specific sections of the original block’s index is a computationally easy operation, the index-header is never uploaded back to the object storage and multiple store-gateway instances (or the same instance after a rolling update without a persistent disk) will re-build the index-header from original block’s index each time, if not already existing on local disk.\nFormat (version 1) The index-header is a subset of the block index and contains:\nSymbol Table, used to unintern string values Posting Offset Table, used to lookup postings The following describes the format of the index-header file found in each block store-gateway local directory. It is terminated by a table of contents which serves as an entry point into the index.\n┌─────────────────────────────┬───────────────────────────────┐ │ magic(0xBAAAD792) \u003c4b\u003e │ version(1) \u003c1 byte\u003e │ ├─────────────────────────────┬───────────────────────────────┤ │ index version(2) \u003c1 byte\u003e │ index PostingOffsetTable \u003c8b\u003e │ ├─────────────────────────────┴───────────────────────────────┤ │ ┌─────────────────────────────────────────────────────────┐ │ │ │ Symbol Table (exact copy from original index) │ │ │ ├─────────────────────────────────────────────────────────┤ │ │ │ Posting Offset Table (exact copy from index) │ │ │ ├─────────────────────────────────────────────────────────┤ │ │ │ TOC │ │ │ └─────────────────────────────────────────────────────────┘ │ └─────────────────────────────────────────────────────────────┘ ","categories":"","description":"","excerpt":"In order to query series inside blocks from object storage, the …","ref":"/docs/blocks-storage/binary-index-header/","tags":"","title":"Binary index-header"},{"body":"The bucket index is a per-tenant file containing the list of blocks and block deletion marks in the storage. The bucket index itself is stored in the backend object storage, is periodically updated by the compactor, and used by queriers, store-gateways and rulers to discover blocks in the storage.\nThe bucket index usage is optional and can be enabled via -blocks-storage.bucket-store.bucket-index.enabled=true (or its respective YAML config option).\nBenefits The querier, store-gateway and ruler need to have an almost up-to-date view over the entire storage bucket, in order to find the right blocks to lookup at query time (querier) and load block’s index-header (store-gateway). Because of this, they need to periodically scan the bucket to look for new blocks uploaded by ingester or compactor, and blocks deleted (or marked for deletion) by compactor.\nWhen the bucket index is enabled, the querier, store-gateway and ruler periodically look up the per-tenant bucket index instead of scanning the bucket via “list objects” operations. This brings few benefits:\nReduced number of API calls to the object storage by querier and store-gateway No “list objects” storage API calls done by querier and store-gateway The querier is up and running immediately after the startup (no need to run an initial bucket scan) Structure of the index The bucket-index.json.gz contains:\nblocks\nList of complete blocks of a tenant, including blocks marked for deletion (partial blocks are excluded from the index). block_deletion_marks\nList of block deletion marks. updated_at\nUnix timestamp (seconds precision) of when the index has been updated (written in the storage) the last time. How it gets updated The compactor periodically scans the bucket and uploads an updated bucket index to the storage. The frequency at which the bucket index is updated can be configured via -compactor.cleanup-interval.\nDespite using the bucket index is optional, the index itself is built and updated by the compactor even if -blocks-storage.bucket-store.bucket-index.enabled has not been enabled. This is intentional, so that once a Cortex cluster operator decides to enable the bucket index in a live cluster, the bucket index for any tenant is already existing and query results consistency is guaranteed. The overhead introduced by keeping the bucket index updated is expected to be non significative.\nHow it’s used by the querier At query time the querier and ruler check whether the bucket index for the tenant has already been loaded in memory. If not, the querier and ruler download it from the storage and cache it in memory.\nGiven it’s a small file, lazy downloading it doesn’t significantly impact on first query performances, but allows to get a querier up and running without pre-downloading every tenant’s bucket index. Moreover, if the metadata cache is enabled, the bucket index will be cached for a short time in a shared cache, reducing the actual latency and number of API calls to the object storage in case multiple queriers and rulers will fetch the same tenant’s bucket index in a short time.\nWhile in-memory, a background process will keep it updated at periodic intervals, so that subsequent queries from the same tenant to the same querier instance will use the cached (and periodically updated) bucket index. There are two config options involved:\n-blocks-storage.bucket-store.sync-interval\nThis option configures how frequently a cached bucket index should be refreshed. -blocks-storage.bucket-store.bucket-index.update-on-error-interval\nIf downloading a bucket index fails, the failure is cached for a short time in order to avoid hammering the backend storage. This option configures how frequently a bucket index, which previously failed to load, should be tried to load again. If a bucket index is unused for a long time (configurable via -blocks-storage.bucket-store.bucket-index.idle-timeout), e.g. because that querier instance is not receiving any query from the tenant, the querier will offload it, stopping to keep it updated at regular intervals. This is particularly for tenants which are resharded to different queriers when shuffle sharding is enabled.\nFinally, at query time the querier and ruler check how old a bucket index is (based on its updated_at) and fail a query if its age is older than -blocks-storage.bucket-store.bucket-index.max-stale-period. This circuit breaker is used to ensure queriers and rulers will not return any partial query results due to a stale view over the long-term storage.\nHow it’s used by the store-gateway The store-gateway, at startup and periodically, fetches the bucket index for each tenant belonging to their shard and uses it as the source of truth for the blocks (and deletion marks) in the storage. This removes the need to periodically scan the bucket to discover blocks belonging to their shard.\n","categories":"","description":"","excerpt":"The bucket index is a per-tenant file containing the list of blocks …","ref":"/docs/blocks-storage/bucket-index/","tags":"","title":"Bucket Index"},{"body":"","categories":"","description":"","excerpt":"","ref":"/docs/configuration/","tags":"","title":"Configuration"},{"body":"Cortex integration tests are written in Go and based on a custom framework running Cortex and its dependencies in Docker containers and using the Go testing package for assertions. Integration tests run in CI for every PR, and can be easily executed locally during development (it just requires Docker).\nHow to run integration tests When integration tests run in CI, we build the Cortex docker image based on the PR code and then run the integration tests against it. When running tests locally you should build the Cortex Docker image first:\nmake ./cmd/cortex/.uptodate This will locally build the quay.io/cortexproject/cortex:latest image used by integration tests. Whenever the Cortex code changes (cmd/, pkg/ or vendors) you should rebuild the Cortex image, while it’s not necessary to rebuild it while developing integration tests.\nOnce the Docker image is built, you can run integration tests:\ngo test -v -tags=integration,requires_docker,integration_alertmanager,integration_memberlist,integration_querier,integration_ruler,integration_query_fuzz ./integration/... If you want to run a single test you can use a filter. For example, to only run TestRulerAPISharding:\ngo test -v -tags=integration,integration_ruler -timeout 2400s -v -count=1 ./integration/... -run \"^TestRulerAPISharding$\" Supported environment variables CORTEX_IMAGE\nDocker image used to run Cortex in integration tests (defaults to quay.io/cortexproject/cortex:latest) CORTEX_CHECKOUT_DIR\nThe absolute path of the Cortex repository local checkout (defaults to $GOPATH/src/github.com/cortexproject/cortex) E2E_TEMP_DIR\nThe absolute path to a directory where the integration test will create an additional temporary directory to store files generated during the test. E2E_NETWORK_NAME\nName of the docker network to create and use for integration tests. If no variable is set, defaults to e2e-cortex-test. The requires_docker tag Integration tests have requires_docker tag (// +build requires_docker line followed by empty line on top of Go files), to avoid running them unintentionally as they require Docker, e.g. by running go test ./... in main Cortex package.\nIsolation Each integration test runs in isolation. For each integration test, we do create a Docker network, start Cortex and its dependencies containers, push/query series to/from Cortex and run assertions on it. Once the test has done, both the Docker network and containers are terminated and deleted.\n","categories":"","description":"","excerpt":"Cortex integration tests are written in Go and based on a custom …","ref":"/docs/contributing/how-integration-tests-work/","tags":"","title":"How integration tests work"},{"body":"The procedure is:\nUpdate build-image/Dockerfile Create a PR to master with that changed, after the PR is merged to master, the new build image is available in the quay.io repository. Check github action logs here for to find the image tag. Create another PR to replace the image tag in .github/workflows/* (there may be multiple references) and Makefile (variable LATEST_BUILD_IMAGE_TAG). If you are updating Go’s runtime version be sure to change actions/setup-go’s go-version in ``.github/workflows/*`. Open a PR and make sure the CI with new build-image passes ","categories":"","description":"","excerpt":"The procedure is:\nUpdate build-image/Dockerfile Create a PR to master …","ref":"/docs/contributing/how-to-update-the-build-image/","tags":"","title":"How to update the build image"},{"body":"This article describes how to migrate existing Cortex cluster from chunks storage to blocks storage, and highlight possible issues you may encounter in the process.\nThis document replaces the Cortex proposal, which was written before support for migration was in place.\nIntroduction This article assumes that:\nCortex cluster is managed by Kubernetes Cortex is using chunks storage Ingesters are using WAL Cortex version 1.4.0 or later. If your ingesters are not using WAL, the documented procedure will still apply, but the presented migration script will not work properly without changes, as it assumes that ingesters are managed via StatefulSet.\nThe migration procedure is composed by 3 steps:\nPreparation Ingesters migration Cleanup In case of any issue during or after the migration, this document also outlines a Rollback strategy.\nStep 1: Preparation Before starting the migration of ingesters, we need to prepare other services.\nQuerier and Ruler Everything discussed for querier applies to ruler as well, since it shares querier configuration – CLI flags prefix is -querier even when used by ruler.\nQuerier and ruler need to be reconfigured as follow:\n-querier.second-store-engine=blocks -querier.query-store-after=0 -querier.second-store-engine=blocks Querier (and ruler) needs to be reconfigured to query both chunks storage and blocks storage at the same time. This is achieved by using -querier.second-store-engine=blocks option, and providing querier with full blocks configuration, but keeping “primary” store set to -store.engine=chunks.\n-querier.query-store-after=0 Querier (and ruler) has an option -querier.query-store-after to query store only if query hits data older than some period of time. For example, if ingesters keep 12h of data in memory, there is no need to hit the store for queries that only need last 1h of data. During the migration, this flag needs to be set to 0, to make queriers always consult the store when handling queries. As chunks ingesters shut down, they flush chunks to the storage. They are then replaced with new ingesters configured to use blocks. Queriers cannot fetch recent chunks from ingesters directly (as blocks ingester don’t reload chunks), and need to use storage instead.\nCompactor and Store-gateway Compactor and store-gateway services should be deployed and successfully up and running before migrating ingesters.\nIngester – blocks Migration script presented in Step 2 assumes that there are two StatefulSets of ingesters: existing one configured with chunks, and the new one with blocks. New StatefulSet with blocks ingesters should have 0 replicas at the beginning of migration.\nTable-Manager - chunks If you use a store with provisioned IO, e.g. DynamoDB, scale up the provision before starting the migration. Each ingester will need to flush all chunks before exiting, so will write to the store at many times the normal rate.\nStop or reconfigure the table-manager to stop it adjusting the provision back to normal. (Don’t do the migration on Wednesday night when a new weekly table might be required.)\nStep 2: Ingesters migration We have developed a script available in Cortex tools/migrate-ingester-statefulsets.sh to migrate ingesters between two StatefulSets, shutting down ingesters one by one.\nIt can be used like this:\n$ tools/migrate-ingester-statefulsets.sh \u003cnamespace\u003e \u003cingester-old\u003e \u003cingester-new\u003e \u003cnum-instances\u003e Where parameters are:\n\u003cnamespace\u003e: Kubernetes namespace where the Cortex cluster is running \u003cingester-old\u003e: name of the ingesters StatefulSet to scale down (running chunks storage) \u003cingester-new\u003e: name of the ingesters StatefulSet to scale up (running blocks storage) \u003cnum-instances\u003e: number of instances to scale down (in ingester-old statefulset) and scale up (in ingester-new), or “all” – which will scale down all remaining instances in ingester-old statefulset After starting new pod in ingester-new statefulset, script then triggers /shutdown endpoint on the old ingester. When the flushing on the old ingester is complete, scale down of statefulset continues, and process repeats.\nThe script supports both migration from chunks to blocks, and vice versa (eg. rollback).\nKnown issues There are few known issues with the script:\nIf expected messages don’t appear in the log, but pod keeps on running, the script will never finish. Script doesn’t verify that flush finished without any error. Step 3: Cleanup When the ingesters migration finishes, there are still two StatefulSets, with original StatefulSet (running the chunks storage) having 0 instances now.\nAt this point, we can delete the old StatefulSet and its persistent volumes and recreate it with final blocks storage configuration (eg. changing PVs), and use the script again to move pods from ingester-blocks to ingester.\nQuerier (and ruler) can be reconfigured to use blocks as “primary” store to search, and chunks as secondary:\n-store.engine=blocks -querier.second-store-engine=chunks -querier.use-second-store-before-time=\u003ctimestamp after ingesters migration has completed\u003e -querier.ingester-streaming=true -querier.use-second-store-before-time The CLI flag -querier.use-second-store-before-time (or its respective YAML config option) is only available for secondary store. This flag can be set to a timestamp when migration has finished, and it avoids querying secondary store (chunks) for data when running queries that don’t need data before given time.\nBoth primary and secondary stores are queried before this time, so the overlap where some data is in chunks and some in blocks is covered.\nRollback If rollback to chunks is needed for any reason, it is possible to use the same migration script with reversed arguments:\nScale down ingesters StatefulSet running blocks storage Scale up ingesters StatefulSet running chunks storage Blocks ingesters support the same /shutdown endpoint for flushing data.\nDuring the rollback, queriers and rulers need to use the same configuration changes as during migration. You should also make sure the following settings are applied:\n-store.engine=chunks -querier.second-store-engine=blocks -querier.use-second-store-before-time should not be set -querier.ingester-streaming=false Once the rollback is complete, some configuration changes need to stay in place, because some data has already been stored to blocks:\nThe query sharding in the query-frontend must be kept disabled, otherwise querying blocks will not work correctly store-gateway needs to keep running, otherwise querying blocks will fail compactor may be shutdown, after it has no more compaction work to do Kubernetes resources related to the ingesters running the blocks storage may be deleted.\nKnown issues After rollback, chunks ingesters will replay their old Write-Ahead-Log, thus loading old chunks into memory. WAL doesn’t remember whether these old chunks were already flushed or not, so they will be flushed again to the storage. Until that flush happens, Cortex reports those chunks as unflushed, which may trigger some alerts based on cortex_oldest_unflushed_chunk_timestamp_seconds metric.\nAppendix Jsonnet config This section shows how to use cortex-jsonnet to configure additional services.\nWe will assume that main.jsonnet is main configuration for the cluster, that also imports temp.jsonnet – with our temporary configuration for migration.\nIn main.jsonnet we have something like this:\nlocal cortex = import 'cortex/cortex.libsonnet'; local wal = import 'cortex/wal.libsonnet'; local temp = import 'temp.jsonnet'; // Note that 'tsdb' is not imported here. cortex + wal + temp { _images+:: (import 'images.libsonnet'), _config+:: { cluster: 'k8s-cluster', namespace: 'k8s-namespace', ... To configure querier to use secondary store for querying, we need to add:\nquerier_second_storage_engine: 'blocks', blocks_storage_bucket_name: 'bucket-for-storing-blocks', to the _config object in main.jsonnet.\nLet’s generate blocks configuration now in temp.jsonnet. There are comments inside that should give you an idea about what’s happening. Most important thing is generating resources with blocks configuration, and exposing some of them.\n{ local cortex = import 'cortex/cortex.libsonnet', local tsdb = import 'cortex/tsdb.libsonnet', local rootConfig = self._config, local statefulSet = $.apps.v1beta1.statefulSet, // Prepare TSDB resources, but hide them. Cherry-picked resources will be exposed later. tsdb_config:: cortex + tsdb + { _config+:: { cluster: rootConfig.cluster, namespace: rootConfig.namespace, external_url: rootConfig.external_url, // This Cortex cluster is using the blocks storage. storage_tsdb_bucket_name: rootConfig.storage_tsdb_bucket_name, cortex_store_gateway_data_disk_size: '100Gi', cortex_compactor_data_disk_class: 'fast', }, // We create another statefulset for ingesters here, with different name. ingester_blocks_statefulset: self.newIngesterStatefulSet('ingester-blocks', self.ingester_container) + statefulSet.mixin.spec.withReplicas(0), ingester_blocks_pdb: self.newIngesterPdb('ingester-blocks-pdb', 'ingester-blocks'), ingester_blocks_service: $.util.serviceFor(self.ingester_blocks_statefulset, self.ingester_service_ignored_labels), }, _config+: { queryFrontend+: { // Disabled because querying blocks-data breaks if query is rewritten for sharding. sharded_queries_enabled: false, }, }, // Expose some services from TSDB configuration, needed for running Querier with Chunks as primary and TSDB as secondary store. tsdb_store_gateway_pdb: self.tsdb_config.store_gateway_pdb, tsdb_store_gateway_service: self.tsdb_config.store_gateway_service, tsdb_store_gateway_statefulset: self.tsdb_config.store_gateway_statefulset, tsdb_memcached_metadata: self.tsdb_config.memcached_metadata, tsdb_ingester_statefulset: self.tsdb_config.ingester_blocks_statefulset, tsdb_ingester_pdb: self.tsdb_config.ingester_blocks_pdb, tsdb_ingester_service: self.tsdb_config.ingester_blocks_service, tsdb_compactor_statefulset: self.tsdb_config.compactor_statefulset, // Querier and ruler configuration used during migration, and after. query_config_during_migration:: { // Disable streaming, as it is broken when querying both chunks and blocks ingesters at the same time. 'querier.ingester-streaming': 'false', // query-store-after is required during migration, since new ingesters running on blocks will not load any chunks from chunks-WAL. // All such chunks are however flushed to the store. 'querier.query-store-after': '0', }, query_config_after_migration:: { 'querier.ingester-streaming': 'true', 'querier.query-ingesters-within': '13h', // TSDB ingesters have data for up to 4d. 'querier.query-store-after': '12h', // Can be enabled once blocks ingesters are running for 12h. // Switch TSDB and chunks. TSDB is \"primary\" now so that we can skip querying chunks for old queries. // We can do this, because querier/ruler have both configurations. 'store.engine': 'blocks', 'querier.second-store-engine': 'chunks', 'querier.use-second-store-before-time': '2020-07-28T17:00:00Z', // If migration from chunks finished around 18:10 CEST, no need to query chunk store for queries before this time. }, querier_args+:: self.tsdb_config.blocks_metadata_caching_config + self.query_config_during_migration, // + self.query_config_after_migration, ruler_args+:: self.tsdb_config.blocks_metadata_caching_config + self.query_config_during_migration, // + self.query_config_after_migration, } ","categories":"","description":"","excerpt":"This article describes how to migrate existing Cortex cluster from …","ref":"/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/","tags":"","title":"Migrate Cortex cluster from chunks to blocks"},{"body":"Historically scaling the Cortex query frontend has posed some challenges. This document aims to detail how to use some added configuration parameters to correctly scale the frontend. Note that these instructions apply in both the HA single binary scenario or microservices mode.\nScaling the Query Frontend For every query frontend the querier adds a configurable number of concurrent workers which are each capable of executing a query. Each worker is connected to a single query frontend instance, therefore scaling up the query frontend impacts the amount of work each individual querier is attempting to do at any given time.\nScaling up may cause a querier to attempt more work than they are configured for due to restrictions such as memory and cpu limits. Additionally, the PromQL engine itself is limited in the number of queries it can do as configured by the -querier.max-concurrent parameter. Attempting more queries concurrently than this value causes the queries to queue up in the querier itself.\nFor similar reasons scaling down the query frontend may cause a querier to not use its allocated memory and cpu effectively. This will lower effective resource utilization. Also, because individual queriers will be doing less work, this may cause increased queueing in the query frontends.\nQuerier Max Concurrency To guarantee that querier doesn’t receive more queries that it can handle at the same time, make sure to configure the querier to match its PromQL concurrency with number of connections. This can be done by using -querier.worker-match-max-concurrent=true option, or match_max_concurrent: true field in frontend_worker section of YAML config file. This allows the operator to freely scale the frontend or scheduler up and down without impacting the amount of work an individual querier is attempting to perform.\nQuery Scheduler Query scheduler is a service that moves the in-memory queue from query frontend to a separate component. This makes scaling query frontend easier, as it allows running multiple query frontends without increasing the number of queues.\nIn order to use query scheduler, both query frontend and queriers must be configured with query scheduler address (using -frontend.scheduler-address and -querier.scheduler-address options respectively).\nNote that querier will only fetch queries from query frontend or query scheduler, but not both. -querier.frontend-address and -querier.scheduler-address options are mutually exclusive, and at most one can be set.\nWhen using query scheduler, it is recommended to run two query scheduler instances. Running only one query scheduler poses a risk of increased query latency when single scheduler crashes or restarts. Running two query-schedulers should be enough even for large Cortex clusters with an high QPS.\nWhen using single-binary mode, Cortex defaults to run without query scheduler.\nDNS Configuration / Readiness When a new frontend is first created on scale up it will not immediately have queriers attached to it. The existing endpoint /ready returns HTTP 200 status code only when the query frontend is ready to serve queries. Make sure to configure this endpoint as a healthcheck in your load balancer, otherwise a query frontend scale up event might result in failed queries or high latency for a bit while queriers attach.\nWhen using query frontend with query scheduler, /ready will report 200 status code only after frontend discovers some schedulers via DNS resolution.\n","categories":"","description":"","excerpt":"Historically scaling the Cortex query frontend has posed some …","ref":"/docs/operations/scaling-query-frontend/","tags":"","title":"Scaling the Query Frontend"},{"body":"If you have configured your cluster to write new data to blocks, there is still a question about old data. Cortex can query both chunks and the blocks at the same time, but converting old chunks to blocks still has some benefits, like being able to decommission the chunks storage backend and save costs. This document presents set of tools for doing the conversion.\nOriginal design document for blocksconvert is also available.\nTools Cortex provides a tool called blocksconvert, which is actually collection of three tools for converting chunks to blocks.\nTools are:\nScanner\nScans the chunks index database and produces so-called “plan files”, each file being a set of series and chunks for each series. Plan files are uploaded to the same object store bucket where blocks live. Scheduler\nLooks for plan files, and distributes them to builders. Scheduler has global view of overall conversion progress. Builder\nAsks scheduler for next plan file to work on, fetches chunks, puts them into TSDB block, and uploads the block to the object store. It repeats this process until there are no more plans. Cleaner\nCleaner asks scheduler for next plan file to work on, but instead of building the block, it actually REMOVES CHUNKS and INDEX ENTRIES from the Index database. All tools start HTTP server (see -server.http* options) exposing the /metrics endpoint. All tools also start gRPC server (-server.grpc* options), but only Scheduler exposes services on it.\nScanner Scanner is started by running blocksconvert -target=scanner. Scanner requires configuration for accessing Cortex Index:\n-schema-config-file – this is standard Cortex schema file. -bigtable.instance, -bigtable.project – options for BigTable access. -dynamodb.url - for DynamoDB access. Example dynamodb://us-east-1/ -blocks-storage.backend and corresponding -blocks-storage.* options for storing plan files. -scanner.output-dir – specifies local directory for writing plan files to. Finished plan files are deleted after upload to the bucket. List of scanned tables is also kept in this directory, to avoid scanning the same tables multiple times when Scanner is restarted. -scanner.allowed-users – comma-separated list of Cortex tenants that should have plans generated. If empty, plans for all found users are generated. -scanner.ignore-users-regex - If plans for all users are generated (-scanner.allowed-users is not set), then users matching this non-empty regular expression will be skipped. -scanner.tables-limit – How many tables should be scanned? By default all tables are scanned, but when testing scanner it may be useful to start with small number of tables first. -scanner.tables – Comma-separated list of tables to be scanned. Can be used to scan specific tables only. Note that schema is still used to find all tables first, and then this list is consulted to select only specified tables. -scanner.scan-period-start \u0026 -scanner.scan-period-end - limit the scan to a particular date range (format like 2020-12-31) Scanner will read the Cortex schema file to discover Index tables, and then it will start scanning them from most-recent table first, going back. For each table, it will fully read the table and generate a plan for each user and day stored in the table. Plan files are then uploaded to the configured blocks-storage bucket (at the -blocksconvert.bucket-prefix location prefix), and local copies are deleted. After that, scanner continues with the next table until it scans them all or -scanner.tables-limit is reached.\nNote that even though blocksconvert has options for configuring different Index store backends, it only supports BigTable and DynamoDB at the moment.\nIt is expected that only single Scanner process is running. Scanner does the scanning of multiple table subranges concurrently.\nScanner exposes metrics with cortex_blocksconvert_scanner_ prefix, eg. total number of scanned index entries of different type, number of open files (scanner doesn’t close currently plan files until entire table has been scanned), scanned rows and parsed index entries.\nScanner only supports schema version v9 on DynamoDB; v9, v10 and v11 on BigTable. Earlier schema versions are currently not supported.\nScheduler Scheduler is started by running blocksconvert -target=scheduler. It only needs to be configured with options to access the object store with blocks:\n-blocks-storage.* - Blocks storage object store configuration. -scheduler.scan-interval – How often to scan for plan files and their status. -scheduler.allowed-users – Comma-separated list of Cortex tenants. If set, only plans for these tenants will be offered to Builders. It is expected that only single Scheduler process is running. Schedulers consume very little resources.\nScheduler’s metrics have cortex_blocksconvert_scheduler prefix (number of plans in different states, oldest/newest plan). Scheduler HTTP server also exposes /plans page that shows currently queued plans, and all plans and their status for all users.\nBuilder Builder asks scheduler for next plan to work on, downloads the plan, builds the block and uploads the block to the blocks storage. It then repeats the process while there are still plans.\nBuilder is started by blocksconvert -target=builder. It needs to be configured with Scheduler endpoint, Cortex schema file, chunk-store specific options and blocks storage to upload blocks to.\n-builder.scheduler-endpoint - where to find scheduler, eg. “scheduler:9095” -schema-config-file - Cortex schema file, used to find out which chunks store to use for given plan -gcs.bucketname – when using GCS as chunks store (other chunks backend storages, like S3, are supported as well) -blocks-storage.* - blocks storage configuration -builder.output-dir - Local directory where Builder keeps the block while it is being built. Once block is uploaded to blocks storage, it is deleted from local directory. Multiple builders may run at the same time, each builder will receive different plan to work on from scheduler. Builders are CPU intensive (decoding and merging chunks), and require fast disk IO for writing blocks.\nBuilders’s metrics have cortex_blocksconvert_builder prefix, and include total number of fetched chunks and their size, read position of the current plan and plan size, total number of written series and samples, number of chunks that couldn’t be downloaded.\nCleaner Cleaner is similar to builder in that it asks scheduler for next plan to work on, but instead of building blocks, it actually REMOVES CHUNKS and INDEX ENTRIES. Use with caution.\nCleaner is started by using blocksconvert -target=cleaner. Like Builder, it needs Scheduler endpoint, Cortex schema file, index and chunk-store specific options. Note that Cleaner works with any index store supported by Cortex, not just BigTable.\n-cleaner.scheduler-endpoint – where to find scheduler -blocks-storage.* – blocks storage configuration, used for downloading plan files -cleaner.plans-dir – local directory to store plan file while it is being processed by Cleaner. -schema-config-file – Cortex schema file. Cleaner doesn’t scan for index entries, but uses existing plan files to find chunks and index entries. For each series, Cleaner needs to download at least one chunk. This is because plan file doesn’t contain label names and values, but chunks do. Cleaner will then delete all index entries associated with the series, and also all chunks.\nWARNING: If both Builder and Cleaner run at the same time and use use the same Scheduler, some plans will be handled by builder, and some by cleaner! This will result in a loss of data!\nCleaner should only be deployed if no other Builder is running. Running multiple Cleaners at once is not supported, and will result in leftover chunks and index entries. Reason for this is that chunks can span multiple days, and chunk is fully deleted only when processing plan (day) when chunk started. Since cleaner also needs to download some chunks to be able to clean up all index entries, when using multiple cleaners, it can happen that cleaner processing older plans will delete chunks required to properly clean up data in newer plans. When using single cleaner only, this is not a problem, since scheduler sends plans to cleaner in time-reversed order.\nNote: Cleaner is designed for use in very special cases, eg. when deleting chunks and index entries for a specific customer. If blocksconvert was used to convert ALL chunks to blocks, it is simpler to just drop the index and chunks database afterwards. In such case, Cleaner is not needed.\nLimitations The blocksconvert toolset currently has the following limitations:\nScanner supports only BigTable and DynamoDB for chunks index backend, and cannot currently scan other databases. Supports only chunks schema versions v9 for DynamoDB; v9, v10 and v11 for Bigtable. ","categories":"","description":"","excerpt":"If you have configured your cluster to write new data to blocks, there …","ref":"/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/","tags":"","title":"Convert long-term storage from chunks to blocks"},{"body":"","categories":"","description":"","excerpt":"","ref":"/docs/guides/","tags":"","title":"Guides"},{"body":"For the v1.0 release, we want to provide the following guarantees:\nFlags, Config and minor version upgrades Upgrading cortex from one minor version to the next should “just work”; that being said, we don’t want to bump the major version every time we remove a flag, so we will will keep deprecated flags around for 2 minor release. There is a metric (deprecated_flags_inuse_total) you can alert on to find out if you’re using a deprecated flag.\nSimilarly to flags, minor version upgrades using config files should “just work”. If we do need to change config, we will keep the old way working for two minor version. There will be a metric you can alert on for this too.\nThese guarantees don’t apply for experimental features.\nReading old data The Cortex maintainers commit to ensuring future version of Cortex can read data written by versions up to two years old. In practice we expect to be able to read more, but this is our guarantee.\nAPI Compatibility Cortex strives to be 100% API compatible with Prometheus (under /prometheus/* and /api/prom/*); any deviation from this is considered a bug, except:\nAdditional API endpoints for creating, removing and modifying alerts and recording rules. Additional API around pushing metrics (under /api/push). Additional API endpoints for management of Cortex itself, such as the ring. These APIs are not part of the any compatibility guarantees. For more information, please refer to the limitations doc.\nExperimental features Cortex is an actively developed project and we want to encourage the introduction of new features and capability. As such, not everything in each release of Cortex is considered “production-ready”. We don’t provide any backwards compatibility guarantees on these and the config and flags might break.\nCurrently experimental features are:\nRuler: Evaluate rules to query frontend instead of ingesters (enabled via -ruler.frontend-address ) S3 Server Side Encryption (SSE) using KMS (including per-tenant KMS config overrides). Azure blob storage. Zone awareness based replication. Ruler API (to PUT rules). Alertmanager: API (enabled via -experimental.alertmanager.enable-api) Sharding of tenants across multiple instances (enabled via -alertmanager.sharding-enabled) Receiver integrations firewall (configured via -alertmanager.receivers-firewall.*) Memcached client DNS-based service discovery. In-memory (FIFO) and Redis cache. gRPC Store. TLS configuration in gRPC and HTTP clients. TLS configuration in Etcd client. Blocksconvert tools OpenStack Swift storage support. Metric relabeling in the distributor. Scalable query-frontend (when using query-scheduler) Ingester: do not unregister from ring on shutdown (-ingester.unregister-on-shutdown=false) Distributor: Do not extend writes on unhealthy ingesters (-distributor.extend-writes=false) Accept multiple HA pairs in the same request (enabled via -experimental.distributor.ha-tracker.mixed-ha-samples=true) Tenant Deletion in Purger, for blocks storage. Query-frontend: query stats tracking (-frontend.query-stats-enabled) Blocks storage bucket index The bucket index support in the querier and store-gateway (enabled via -blocks-storage.bucket-store.bucket-index.enabled=true) is experimental The block deletion marks migration support in the compactor (-compactor.block-deletion-marks-migration-enabled) is temporarily and will be removed in future versions Querier: tenant federation The thanosconvert tool for converting Thanos block metadata to Cortex HA Tracker: cleanup of old replicas from KV Store. Instance limits in ingester and distributor Exemplar storage, currently in-memory only within the Ingester based on Prometheus exemplar storage (-blocks-storage.tsdb.max-exemplars) Querier limits: -querier.max-fetched-chunks-per-query -querier.max-fetched-chunk-bytes-per-query -querier.max-fetched-series-per-query Alertmanager limits notification rate (-alertmanager.notification-rate-limit and -alertmanager.notification-rate-limit-per-integration) dispatcher groups (-alertmanager.max-dispatcher-aggregation-groups) user config size (-alertmanager.max-config-size-bytes) templates count in user config (-alertmanager.max-templates-count) max template size (-alertmanager.max-template-size-bytes) Disabling ring heartbeat timeouts -distributor.ring.heartbeat-timeout=0 -ring.heartbeat-timeout=0 -ruler.ring.heartbeat-timeout=0 -alertmanager.sharding-ring.heartbeat-timeout=0 -compactor.ring.heartbeat-timeout=0 -store-gateway.sharding-ring.heartbeat-timeout=0 Disabling ring heartbeats -distributor.ring.heartbeat-period=0 -ingester.heartbeat-period=0 -ruler.ring.heartbeat-period=0 -alertmanager.sharding-ring.heartbeat-period=0 -compactor.ring.heartbeat-period=0 -store-gateway.sharding-ring.heartbeat-period=0 Compactor shuffle sharding Enabled via -compactor.sharding-enabled=true, -compactor.sharding-strategy=shuffle-sharding, and -compactor.tenant-shard-size set to a value larger than 0. Vertical sharding at query frontend for range/instant queries -frontend.query-vertical-shard-size (int) CLI flag query_vertical_shard_size (int) field in runtime config file Snapshotting of in-memory TSDB on disk during shutdown -blocks-storage.tsdb.memory-snapshot-on-shutdown (boolean) CLI flag Out of order samples support -blocks-storage.tsdb.out-of-order-cap-max (int) CLI flag -ingester.out-of-order-time-window (duration) CLI flag out_of_order_time_window (duration) field in runtime config file Store Gateway Zone Stable Shuffle Sharding -store-gateway.sharding-ring.zone-stable-shuffle-sharding CLI flag zone_stable_shuffle_sharding (boolean) field in config file Basic Lifecycler (Storegateway, Alertmanager, Ruler) Final Sleep on shutdown, which tells the pod wait before shutdown, allowing a delay to propagate ring changes. -ruler.ring.final-sleep (duration) CLI flag store-gateway.sharding-ring.final-sleep (duration) CLI flag alertmanager-sharding-ring.final-sleep (duration) CLI flag OTLP Receiver Persistent tokens in the Ruler Ring: -ruler.ring.tokens-file-path (path) CLI flag Native Histograms Ingestion can be enabled by setting -blocks-storage.tsdb.enable-native-histograms=true on Ingester. String interning for metrics labels Enable string interning for metrics labels by setting -ingester.labels-string-interning-enabled on Ingester. Query-frontend: query rejection (-frontend.query-rejection.enabled) Querier: protobuf codec (-api.querier-default-codec) ","categories":"","description":"","excerpt":"For the v1.0 release, we want to provide the following guarantees: …","ref":"/docs/configuration/v1guarantees/","tags":"","title":"v1.x Guarantees"},{"body":"Cortex exposes an HTTP API for pushing and querying time series data, and operating the cluster itself.\nFor the sake of clarity, in this document we have grouped API endpoints by service, but keep in mind that they’re exposed both when running Cortex in microservices and singly-binary mode:\nMicroservices: each service exposes its own endpoints Single-binary: the Cortex process exposes all API endpoints for the services running internally Endpoints API Service Available Since Endpoint Index page All services GET / Configuration All services GET /config Runtime Configuration All services GET /runtime_config Services status All services GET /services Readiness probe All services GET /ready Metrics All services GET /metrics Pprof All services GET /debug/pprof Fgprof All services GET /debug/fgprof Remote write Distributor POST /api/v1/push OTLP receiver Distributor POST /api/v1/otlp/v1/metrics Tenants stats Distributor GET /distributor/all_user_stats HA tracker status Distributor GET /distributor/ha_tracker Flush blocks Ingester GET,POST /ingester/flush Shutdown Ingester GET,POST /ingester/shutdown Ingesters ring status Ingester GET /ingester/ring Ingester tenants stats Ingester GET /ingester/all_user_stats Ingester mode Ingester GET,POST /ingester/mode Instant query Querier, Query-frontend GET,POST \u003cprometheus-http-prefix\u003e/api/v1/query Range query Querier, Query-frontend GET,POST \u003cprometheus-http-prefix\u003e/api/v1/query_range Exemplar query Querier, Query-frontend GET,POST \u003cprometheus-http-prefix\u003e/api/v1/query_exemplars Get series by label matchers Querier, Query-frontend GET,POST \u003cprometheus-http-prefix\u003e/api/v1/series Get label names Querier, Query-frontend GET,POST \u003cprometheus-http-prefix\u003e/api/v1/labels Get label values Querier, Query-frontend GET \u003cprometheus-http-prefix\u003e/api/v1/label/{name}/values Get metric metadata Querier, Query-frontend GET \u003cprometheus-http-prefix\u003e/api/v1/metadata Remote read Querier, Query-frontend POST \u003cprometheus-http-prefix\u003e/api/v1/read Build information Querier, Query-frontend v1.15.0 GET \u003cprometheus-http-prefix\u003e/api/v1/status/buildinfo Get tenant ingestion stats Querier GET /api/v1/user_stats Ruler ring status Ruler GET /ruler/ring Ruler rules Ruler GET /ruler/rule_groups List rules Ruler GET \u003cprometheus-http-prefix\u003e/api/v1/rules List alerts Ruler GET \u003cprometheus-http-prefix\u003e/api/v1/alerts List rule groups Ruler GET /api/v1/rules Get rule groups by namespace Ruler GET /api/v1/rules/{namespace} Get rule group Ruler GET /api/v1/rules/{namespace}/{groupName} Set rule group Ruler POST /api/v1/rules/{namespace} Delete rule group Ruler DELETE /api/v1/rules/{namespace}/{groupName} Delete namespace Ruler DELETE /api/v1/rules/{namespace} Delete tenant configuration Ruler POST /ruler/delete_tenant_config Alertmanager status Alertmanager GET /multitenant_alertmanager/status Alertmanager configs Alertmanager GET /multitenant_alertmanager/configs Alertmanager ring status Alertmanager GET /multitenant_alertmanager/ring Alertmanager UI Alertmanager GET /\u003calertmanager-http-prefix\u003e Alertmanager Delete Tenant Configuration Alertmanager POST /multitenant_alertmanager/delete_tenant_config Get Alertmanager configuration Alertmanager GET /api/v1/alerts Set Alertmanager configuration Alertmanager POST /api/v1/alerts Delete Alertmanager configuration Alertmanager DELETE /api/v1/alerts Tenant delete request Purger POST /purger/delete_tenant Tenant delete status Purger GET /purger/delete_tenant_status Store-gateway ring status Store-gateway GET /store-gateway/ring Compactor ring status Compactor GET /compactor/ring Get rule files Configs API (deprecated) GET /api/prom/configs/rules Set rule files Configs API (deprecated) POST /api/prom/configs/rules Get template files Configs API (deprecated) GET /api/prom/configs/templates Set template files Configs API (deprecated) POST /api/prom/configs/templates Get Alertmanager config file Configs API (deprecated) GET /api/prom/configs/alertmanager Set Alertmanager config file Configs API (deprecated) POST /api/prom/configs/alertmanager Validate Alertmanager config Configs API (deprecated) POST /api/prom/configs/alertmanager/validate Deactivate configs Configs API (deprecated) DELETE /api/prom/configs/deactivate Restore configs Configs API (deprecated) POST /api/prom/configs/restore Path prefixes In this documentation you will find the usage of some placeholders for the path prefixes, whenever the prefix is configurable. The following table shows the supported prefixes.\nPrefix Default CLI Flag YAML Config \u003clegacy-http-prefix\u003e /api/prom -http.prefix http_prefix \u003cprometheus-http-prefix\u003e /prometheus -http.prometheus-http-prefix api \u003e prometheus_http_prefix \u003calertmanager-http-prefix\u003e /alertmanager -http.alertmanager-http-prefix api \u003e alertmanager_http_prefix Authentication When multi-tenancy is enabled, endpoints requiring authentication are expected to be called with the X-Scope-OrgID HTTP request header set to the tenant ID. Otherwise, when multi-tenancy is disabled, Cortex doesn’t require any request to have the X-Scope-OrgID header.\nMulti-tenancy can be enabled/disabled via the CLI flag -auth.enabled or its respective YAML config option.\nFor more information, please refer to the dedicated Authentication and Authorisation guide.\nAll services The following API endpoints are exposed by all services.\nIndex page GET / Displays an index page with links to other web pages exposed by Cortex.\nConfiguration GET /config Displays the configuration currently applied to Cortex (in YAML format), including default values and settings via CLI flags. Sensitive data is masked. Please be aware that the exported configuration doesn’t include the per-tenant overrides.\nDifferent modes GET /config?mode=diff Displays the configuration currently applied to Cortex (in YAML format) as before, but containing only the values that differ from the default values.\nGET /config?mode=defaults Displays the configuration using only the default values.\nRuntime Configuration GET /runtime_config Displays the runtime configuration currently applied to Cortex (in YAML format), including default values. Please be aware that the endpoint will be only available if Cortex is configured with the -runtime-config.file option.\nDifferent modes GET /runtime_config?mode=diff Displays the runtime configuration currently applied to Cortex (in YAML format) as before, but containing only the values that differ from the default values.\nServices status GET /services Displays a web page with the status of internal Cortex services.\nReadiness probe GET /ready Returns 200 when Cortex is ready to serve traffic.\nMetrics GET /metrics Returns the metrics for the running Cortex service in the Prometheus exposition format.\nPprof GET /debug/pprof/heap GET /debug/pprof/block GET /debug/pprof/profile GET /debug/pprof/trace GET /debug/pprof/goroutine GET /debug/pprof/mutex Returns the runtime profiling data in the format expected by the pprof visualization tool. There are many things which can be profiled using this including heap, trace, goroutine, etc.\nFor more information, please check out the official documentation of pprof.\nFgprof GET /debug/fgprof Returns the sampling Go profiling data which allows you to analyze On-CPU as well as Off-CPU (e.g. I/O) time together.\nFor more information, please check out the official documentation of fgprof.\nDistributor Remote write POST /api/v1/push # Legacy POST \u003clegacy-http-prefix\u003e/push Entrypoint for the Prometheus remote write.\nThis API endpoint accepts an HTTP POST request with a body containing a request encoded with Protocol Buffers and compressed with Snappy. The definition of the protobuf message can be found in cortex.proto. The HTTP request should contain the header X-Prometheus-Remote-Write-Version set to 0.1.0.\nFor more information, please check out Prometheus Remote storage integrations.\nRequires authentication.\nOTLP Receiver POST /api/v1/otlp/v1/metrics Entrypoint for the OTLP Receiver\nThis API endpoint accepts a HTTP POST request using OTLP format\nRequires authentication.\nDistributor ring status GET /distributor/ring Displays a web page with the distributor hash ring status, including the state, healthy and last heartbeat time of each distributor.\nTenants stats GET /distributor/all_user_stats # Legacy GET /all_user_stats Displays a web page with per-tenant statistics updated in realtime, including the total number of loaded blocks and active series across all ingesters as well as the current ingestion rate (samples / sec).\nHA tracker status GET /distributor/ha_tracker # Legacy GET /ha-tracker Displays a web page with the current status of the HA tracker, including the elected replica for each Prometheus HA cluster.\nIngester Flush blocks GET,POST /ingester/flush # Legacy GET,POST /flush Triggers a flush of the in-memory time series data to the long-term storage. This endpoint triggers the flush also when -ingester.flush-on-shutdown-with-wal-enabled or -blocks-storage.tsdb.flush-blocks-on-shutdown are disabled.\nThis endpoint accepts tenant parameter to specify tenant whose blocks are compacted and shipped. This parameter may be specified multiple times to select more tenants. If no tenant is specified, all tenants are flushed.\nFlush endpoint now also accepts wait=true parameter, which makes the call synchronous – it will only return after flushing has finished. Note that returned status code does not reflect the result of flush operation.\nShutdown GET,POST /ingester/shutdown # Legacy GET,POST /shutdown Flushes in-memory time series data from ingester to the long-term storage, and shuts down the ingester service. Notice that the other Cortex services are still running, and the operator (or any automation) is expected to terminate the process with a SIGINT / SIGTERM signal after the shutdown endpoint returns. In the meantime, /ready will not return 200. This endpoint will unregister the ingester from the ring even if -ingester.unregister-on-shutdown is disabled.\nThis API endpoint is usually used by scale down automations.\nIngesters ring status GET /ingester/ring # Legacy GET /ring Displays a web page with the ingesters hash ring status, including the state, healthy and last heartbeat time of each ingester.\nIngester tenants stats GET /ingester/all_user_stats Displays a web page with per-tenant statistics updated in realtime, including the total number of loaded blocks and active series from a specific ingester as well as the current ingestion rate (samples / sec).\nIngester mode GET,POST /ingester/mode Change ingester mode between ACTIVE or READONLY. READONLY ingester does not receive push requests and will only be called for query operations.\nThe endpoint accept query param mode or POST as application/x-www-form-urlencoded with mode type.\nQuerier / Query-frontend The following endpoints are exposed both by the querier and query-frontend.\nInstant query GET,POST \u003cprometheus-http-prefix\u003e/api/v1/query # Legacy GET,POST \u003clegacy-http-prefix\u003e/api/v1/query Prometheus-compatible instant query endpoint.\nFor more information, please check out the Prometheus instant query documentation.\nRequires authentication.\nRange query GET,POST \u003cprometheus-http-prefix\u003e/api/v1/query_range # Legacy GET,POST \u003clegacy-http-prefix\u003e/api/v1/query_range Prometheus-compatible range query endpoint. When the request is sent through the query-frontend, the query will be accelerated by query-frontend (results caching and execution parallelisation).\nFor more information, please check out the Prometheus range query documentation.\nRequires authentication.\nExemplar query GET,POST \u003cprometheus-http-prefix\u003e/api/v1/query_exemplars # Legacy GET,POST \u003clegacy-http-prefix\u003e/api/v1/query_exemplars Prometheus-compatible exemplar query endpoint.\nFor more information, please check out the Prometheus exemplar query documentation.\nRequires authentication.\nGet series by label matchers GET,POST \u003cprometheus-http-prefix\u003e/api/v1/series # Legacy GET,POST \u003clegacy-http-prefix\u003e/api/v1/series Find series by label matchers. Starting from release v1.18.0, Cortex by default honors the start and end request parameters and fetches series from either ingester, store gateway or both. The special case is that if start param is not specified, Cortex currently fetches series from data stored in the ingesters.\nFor more information, please check out the Prometheus series endpoint documentation.\nRequires authentication.\nGet label names GET,POST \u003cprometheus-http-prefix\u003e/api/v1/labels # Legacy GET,POST \u003clegacy-http-prefix\u003e/api/v1/labels Get label names of ingested series. Starting from release v1.18.0, Cortex by default honors the start and end request parameters and fetches label names from either ingester, store gateway or both.\nFor more information, please check out the Prometheus get label names documentation.\nRequires authentication.\nGet label values GET \u003cprometheus-http-prefix\u003e/api/v1/label/{name}/values # Legacy GET \u003clegacy-http-prefix\u003e/api/v1/label/{name}/values Get label values for a given label name. Starting from release v1.18.0, Cortex by default honors the start and end request parameters and fetches label values from either ingester, store gateway or both.\nFor more information, please check out the Prometheus get label values documentation.\nRequires authentication.\nGet metric metadata GET \u003cprometheus-http-prefix\u003e/api/v1/metadata # Legacy GET \u003clegacy-http-prefix\u003e/api/v1/metadata Prometheus-compatible metric metadata endpoint.\nFor more information, please check out the Prometheus metric metadata documentation.\nRequires authentication.\nRemote read POST \u003cprometheus-http-prefix\u003e/api/v1/read # Legacy POST \u003clegacy-http-prefix\u003e/api/v1/read Prometheus-compatible remote read endpoint.\nFor more information, please check out Prometheus Remote storage integrations.\nRequires authentication.\nBuild Information GET \u003cprometheus-http-prefix\u003e/api/v1/status/buildinfo # Legacy GET \u003clegacy-http-prefix\u003e/api/v1/status/buildinfo Prometheus-compatible build information endpoint.\nRequires authentication.\nQuerier Get tenant ingestion stats GET /api/v1/user_stats # Legacy GET \u003clegacy-http-prefix\u003e/user_stats Returns realtime ingestion rate, for the authenticated tenant, in JSON format.\nRequires authentication.\nRuler The ruler API endpoints require to configure a backend object storage to store the recording rules and alerts. The ruler API uses the concept of a “namespace” when creating rule groups. This is a stand in for the name of the rule file in Prometheus and rule groups must be named uniquely within a namespace.\nRuler ring status GET /ruler/ring # Legacy GET /ruler_ring Displays a web page with the ruler hash ring status, including the state, healthy and last heartbeat time of each ruler.\nRuler rules GET /ruler/rule_groups List all tenant rules. This endpoint is not part of ruler-API and is always available regardless of whether ruler-API is enabled or not. It should not be exposed to end users. This endpoint returns a YAML dictionary with all the rule groups for each tenant and 200 status code on success.\nList rules GET \u003cprometheus-http-prefix\u003e/api/v1/rules # Legacy GET \u003clegacy-http-prefix\u003e/api/v1/rules Prometheus-compatible rules endpoint to list alerting and recording rules that are currently loaded.\nFor more information, please check out the Prometheus rules documentation.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nList alerts GET \u003cprometheus-http-prefix\u003e/api/v1/alerts # Legacy GET \u003clegacy-http-prefix\u003e/api/v1/alerts Prometheus-compatible rules endpoint to list of all active alerts.\nFor more information, please check out the Prometheus alerts documentation.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nList rule groups GET /api/v1/rules # Legacy GET \u003clegacy-http-prefix\u003e/rules List all rules configured for the authenticated tenant. This endpoint returns a YAML dictionary with all the rule groups for each namespace and 200 status code on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nExample response --- \u003cnamespace1\u003e: - name: \u003cstring\u003e interval: \u003cduration;optional\u003e rules: - record: \u003cstring\u003e expr: \u003cstring\u003e - alert: \u003cstring\u003e expr: \u003cstring\u003e for: \u003cduration\u003e annotations: \u003cannotation_name\u003e: \u003cstring\u003e labels: \u003clabel_name\u003e: \u003cstring\u003e - name: \u003cstring\u003e interval: \u003cduration;optional\u003e rules: - record: \u003cstring\u003e expr: \u003cstring\u003e - alert: \u003cstring\u003e expr: \u003cstring\u003e for: \u003cduration\u003e annotations: \u003cannotation_name\u003e: \u003cstring\u003e labels: \u003clabel_name\u003e: \u003cstring\u003e \u003cnamespace2\u003e: - name: \u003cstring\u003e interval: \u003cduration;optional\u003e rules: - record: \u003cstring\u003e expr: \u003cstring\u003e - alert: \u003cstring\u003e expr: \u003cstring\u003e for: \u003cduration\u003e annotations: \u003cannotation_name\u003e: \u003cstring\u003e labels: \u003clabel_name\u003e: \u003cstring\u003e Get rule groups by namespace GET /api/v1/rules/{namespace} # Legacy GET \u003clegacy-http-prefix\u003e/rules/{namespace} Returns the rule groups defined for a given namespace.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nExample response name: \u003cstring\u003e interval: \u003cduration;optional\u003e rules: - record: \u003cstring\u003e expr: \u003cstring\u003e - alert: \u003cstring\u003e expr: \u003cstring\u003e for: \u003cduration\u003e annotations: \u003cannotation_name\u003e: \u003cstring\u003e labels: \u003clabel_name\u003e: \u003cstring\u003e Get rule group GET /api/v1/rules/{namespace}/{groupName} # Legacy GET \u003clegacy-http-prefix\u003e/rules/{namespace}/{groupName} Returns the rule group matching the request namespace and group name.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nSet rule group POST /api/v1/rules/{namespace} # Legacy POST \u003clegacy-http-prefix\u003e/rules/{namespace} Creates or updates a rule group. This endpoint expects a request with Content-Type: application/yaml header and the rules YAML definition in the request body, and returns 202 on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nExample request Request headers:\nContent-Type: application/yaml Request body:\nname: \u003cstring\u003e interval: \u003cduration;optional\u003e rules: - record: \u003cstring\u003e expr: \u003cstring\u003e - alert: \u003cstring\u003e expr: \u003cstring\u003e for: \u003cduration\u003e annotations: \u003cannotation_name\u003e: \u003cstring\u003e labels: \u003clabel_name\u003e: \u003cstring\u003e Delete rule group DELETE /api/v1/rules/{namespace}/{groupName} # Legacy DELETE \u003clegacy-http-prefix\u003e/rules/{namespace}/{groupName} Deletes a rule group by namespace and group name. This endpoints returns 202 on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nDelete namespace DELETE /api/v1/rules/{namespace} # Legacy DELETE \u003clegacy-http-prefix\u003e/rules/{namespace} Deletes all the rule groups in a namespace (including the namespace itself). This endpoint returns 202 on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.ruler.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nDelete tenant configuration POST /ruler/delete_tenant_config This deletes all rule groups for tenant, and returns 200 on success. Calling endpoint when no rule groups exist for user returns 200. Authentication is only to identify the tenant.\nThis is intended as internal API, and not to be exposed to users. This endpoint is enabled regardless of whether -experimental.ruler.enable-api is enabled or not.\nRequires authentication.\nAlertmanager Alertmanager status GET /multitenant_alertmanager/status # Legacy (microservices mode only) GET /status Displays a web page with the current status of the Alertmanager, including the Alertmanager cluster members.\nAlertmanager configs GET /multitenant_alertmanager/configs List all Alertmanager configurations. This endpoint is not part of alertmanager-API and is always available regardless of whether alertmanager-API is enabled or not. It should not be exposed to end users. This endpoint returns a YAML dictionary with all the Alertmanager configurations and 200 status code on success.\nAlertmanager ring status GET /multitenant_alertmanager/ring Displays a web page with the Alertmanager hash ring status, including the state, healthy and last heartbeat time of each Alertmanager instance.\nAlertmanager UI GET /\u003calertmanager-http-prefix\u003e # Legacy (microservices mode only) GET /\u003clegacy-http-prefix\u003e Displays the Alertmanager UI.\nRequires authentication.\nAlertmanager Delete Tenant Configuration POST /multitenant_alertmanager/delete_tenant_config This endpoint deletes configuration for a tenant identified by X-Scope-OrgID header. It is internal, available even if Alertmanager API is not enabled by using -experimental.alertmanager.enable-api. The endpoint returns a status code of 200 if the user’s configuration has been deleted, or it didn’t exist in the first place.\nRequires authentication.\nGet Alertmanager configuration GET /api/v1/alerts Get the current Alertmanager configuration for the authenticated tenant, reading it from the configured object storage.\nThis endpoint doesn’t accept any URL query parameter and returns 200 on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.alertmanager.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nSet Alertmanager configuration POST /api/v1/alerts Stores or updates the Alertmanager configuration for the authenticated tenant. The Alertmanager configuration is stored in the configured backend object storage.\nThis endpoint expects the Alertmanager YAML configuration in the request body and returns 201 on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.alertmanager.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nNote: When using curl send the request body from a file, ensure that you use the --data-binary flag instead of -d, --data, or --data-ascii. The latter options do not preserve carriage returns and newlines.\nExample request body template_files: default_template: | {{ define \"__alertmanager\" }}AlertManager{{ end }} {{ define \"__alertmanagerURL\" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver | urlquery }}{{ end }} alertmanager_config: | global: smtp_smarthost: 'localhost:25' smtp_from: 'youraddress@example.org' templates: - 'default_template' route: receiver: example-email receivers: - name: example-email email_configs: - to: 'youraddress@example.org' Delete Alertmanager configuration DELETE /api/v1/alerts Deletes the Alertmanager configuration for the authenticated tenant.\nThis endpoint doesn’t accept any URL query parameter and returns 200 on success.\nThis experimental endpoint is disabled by default and can be enabled via the -experimental.alertmanager.enable-api CLI flag (or its respective YAML config option).\nRequires authentication.\nPurger The Purger service provides APIs for requesting deletion of tenants.\nTenant Delete Request POST /purger/delete_tenant Request deletion of ALL tenant data. Only works with blocks storage. Experimental.\nRequires authentication.\nTenant Delete Status GET /purger/delete_tenant_status Returns status of tenant deletion. Output format to be defined. Experimental.\nRequires authentication.\nStore-gateway Store-gateway ring status GET /store-gateway/ring Displays a web page with the store-gateway hash ring status, including the state, healthy and last heartbeat time of each store-gateway.\nCompactor Compactor ring status GET /compactor/ring Displays a web page with the compactor hash ring status, including the state, healthy and last heartbeat time of each compactor.\nConfigs API This service has been deprecated in favour of Ruler and Alertmanager API.\nThe configs API service provides an API-driven multi-tenant approach to handling various configuration files for Prometheus. The service hosts an API where users can read and write Prometheus rule files, Alertmanager configuration files, and Alertmanager templates to a database. Each tenant will have its own set of rule files, Alertmanager config, and templates.\nRequest / response schema The following schema is used both when retrieving the current configs from the API and when setting new configs via the API:\n{ \"id\": 99, \"rule_format_version\": \"2\", \"alertmanager_config\": \"\u003cstandard alertmanager.yaml config\u003e\", \"rules_files\": { \"rules.yaml\": \"\u003cstandard rules.yaml config\u003e\", \"rules2.yaml\": \"\u003cstandard rules.yaml config\u003e\" }, \"template_files\": { \"templates.tmpl\": \"\u003cstandard template file\u003e\", \"templates2.tmpl\": \"\u003cstandard template file\u003e\" } } id\nShould be incremented every time data is updated; Cortex will use the config with the highest number. rule_format_version\nAllows compatibility for tenants with config in Prometheus V1 format. Pass “1” or “2” according to which Prometheus version you want to match. alertmanager_config\nThe contents of the alertmanager config file should be as described here, encoded as a single string to fit within the overall JSON payload. config.rules_files\nThe contents of a rules file should be as described here, encoded as a single string to fit within the overall JSON payload. config.template_files\nThe contents of a template file should be as described here, encoded as a single string to fit within the overall JSON payload. These entries should match the templates entries in alertmanager_config. Example: template_files: myorg.tmpl: | {{ define \"__alertmanager\" }}AlertManager{{ end }} {{ define \"__alertmanagerURL\" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver | urlquery }}{{ end }} alertmanager_config: | templates: - 'myorg.tmpl' Get rule files GET /api/prom/configs/rules Get the current rule files for the authenticated tenant.\nRequires authentication.\nSet rule files POST /api/prom/configs/rules Replace the current rule files for the authenticated tenant.\nRequires authentication.\nGet template files GET /api/prom/configs/templates Get the current template files for the authenticated tenant.\nRequires authentication.\nSet template files POST /api/prom/configs/templates Replace the current template files for the authenticated tenant.\nRequires authentication.\nGet Alertmanager config file GET /api/prom/configs/alertmanager Get the current Alertmanager config for the authenticated tenant.\nRequires authentication.\nSet Alertmanager config file POST /api/prom/configs/alertmanager Replace the current Alertmanager config for the authenticated tenant.\nRequires authentication.\nValidate Alertmanager config file POST /api/prom/configs/alertmanager/validate Validate the Alertmanager config in the request body. The request body is expected to contain only the Alertmanager YAML config.\nDeactivate configs DELETE /api/prom/configs/deactivate Disable configs for the authenticated tenant. Please be aware that setting a new config will effectively “re-enable” the Rules and Alertmanager configuration for the tenant.\nRequires authentication.\nRestore configs POST /api/prom/configs/restore Re-enable configs for the authenticated tenant, after being previously deactivated.\nRequires authentication.\n","categories":"","description":"","excerpt":"Cortex exposes an HTTP API for pushing and querying time series data, …","ref":"/docs/api/","tags":"","title":"HTTP API"},{"body":"The Cortex blocks storage engine stores series in TSDB blocks uploaded in the storage bucket. This makes very easy to migrate the storage from Thanos and/or Prometheus to Cortex, when running the blocks storage.\nCortex blocks storage requirements The Cortex blocks storage has few requirements that should be considered when migrating TSDB blocks from Thanos / Prometheus to Cortex:\nThe blocks in the bucket should be located at bucket://\u003ctenant-id\u003e/\nCortex isolates blocks on a per-tenant basis in the bucket and, for this reason, each tenant blocks should be uploaded to a different location in the bucket. The bucket prefix, where a specific tenant blocks should be uploaded, is /\u003ctenant-id\u003e/; if Cortex is running with auth disabled (no multi-tenancy) then the \u003ctenant-id\u003e to use is fake. Remove Thanos external labels and inject __org_id__ into each block’s meta.json\nEvery block has a little metadata file named meta.json. Thanos stores external labels at thanos \u003e labels, which should be all removed when migrating to Cortex, while the \"__org_id__\": \"\u003ctenant-id\u003e\" added. How to migrate the storage Upload TSDB blocks to Cortex bucket TSDB blocks stored in Prometheus local disk or Thanos bucket should be copied/uploaded to the Cortex bucket at the location bucket://\u003ctenant-id\u003e/ (when Cortex is running with auth disabled then \u003ctenant-id\u003e must be fake).\nMigrate block metadata (meta.json) to Cortex For each block copied/uploaded to the Cortex bucket, there are a few changes required to the meta.json.\nAutomatically migrate metadata using the thanosconvert tool thanosconvert can iterate over a Cortex bucket and make sure that each meta.json has the correct thanos \u003e labels layout.\n⚠ Warning ⚠ thanosconvert will modify files in the bucket you specify. It’s recommended that you have backups or enable object versioning before running this tool.\nTo run thanosconvert, you need to provide it with the bucket configuration in the same format as the blocks storage bucket configuration.\n# bucket-config.yaml backend: s3 s3: endpoint: s3.us-east-1.amazonaws.com bucket_name: my-cortex-bucket You can run thanosconvert directly using Go:\ngo install github.com/cortexproject/cortex/cmd/thanosconvert thanosconvert Or use the provided docker image:\ndocker run quay.io/cortexproject/thanosconvert You can run the tool in dry-run mode first to find out what which blocks it will migrate:\nthanosconvert -config ./bucket-config.yaml -dry-run Once you’re happy with the results, you can run without dry run to migrate blocks:\nthanosconvert -config ./bucket-config.yaml You can cancel a conversion in progress (with Ctrl+C) and rerun thanosconvert. It won’t change any blocks which have been written by Cortex or already converted from Thanos, so you can run thanosconvert multiple times.\nMigrate metadata manually If you need to migrate the block metadata manually, you need to:\nDownload the meta.json to the local filesystem Decode the JSON Manipulate the data structure (see below) Re-encode the JSON Re-upload it to the bucket (overwriting the previous version of the meta.json file) The meta.json should be manipulated in order to ensure:\nIt contains the thanos root-level entry The thanos \u003e labels do not contain any Thanos-specific external label The thanos \u003e labels contain the Cortex-specific external label \"__org_id__\": \"\u003ctenant-id\u003e\" When migrating from Thanos When migrating from Thanos, the easiest approach would be keep the existing thanos root-level entry as is, except:\nCompletely remove the content of thanos \u003e labels Add \"__org_id__\": \"\u003ctenant-id\u003e\" to thanos \u003e labels For example, when migrating a block from Thanos for the tenant user-1, the thanos root-level property within the meta.json file will look like:\n{ \"thanos\": { \"labels\": { \"__org_id__\": \"user-1\" }, \"downsample\": { \"resolution\": 0 }, \"source\": \"compactor\" } } Right now Cortex doesn’t support downsampling so Thanos downsampled blocks are not supported. Downsampled blocks will be simply skipped by the thanosconvert tool.\nIf downsampled blocks are uploaded to the Cortex bucket, they cannot be queried so please exclude them when migrating TSDB blocks.\nWhen migrating from Prometheus When migrating from Prometheus, the meta.json file will not contain any thanos root-level entry and, for this reason, it would need to be generated:\nCreate the thanos root-level entry (see below) Add \"__org_id__\": \"\u003ctenant-id\u003e\" to thanos \u003e labels For example, when migrating a block from Prometheus for the tenant user-1, the thanos root-level property within the meta.json file should be as follow:\n{ \"thanos\": { \"labels\": { \"__org_id__\": \"user-1\" }, \"downsample\": { \"resolution\": 0 }, \"source\": \"compactor\" } } ","categories":"","description":"","excerpt":"The Cortex blocks storage engine stores series in TSDB blocks uploaded …","ref":"/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/","tags":"","title":"Migrate the storage from Thanos and Prometheus"},{"body":"","categories":"","description":"","excerpt":"","ref":"/docs/operations/","tags":"","title":"Operating Cortex"},{"body":"","categories":"","description":"","excerpt":"","ref":"/docs/case-studies/","tags":"","title":"Case Studies"},{"body":"","categories":"","description":"","excerpt":"","ref":"/docs/proposals/","tags":"","title":"Proposals"},{"body":"Context Cortex Alertmanager notification setup follows mostly the syntax of Prometheus Alertmanager since it is based on the same codebase. The following is a description on how to load the configuration setup so that Alertmanager can use it for notification when an alert event happens.\nConfiguring the Cortex Alertmanager storage backend With the introduction of Cortex 1.8, the storage backend config option shifted to the new pattern #3888. You can find the new configuration here\nNote that when using -alertmanager.sharding-enabled=true, the following storage backends are not supported: local, configdb.\nWhen using the new configuration pattern, it is important that any of the old configuration pattern flags are unset (-alertmanager.storage), as well as -\u003cprefix\u003e.configs.url. This is because the old pattern still takes precedence over the new one. The old configuration pattern (-alertmanager.storage) is marked as deprecated and will be removed by Cortex version 1.11. However, this change doesn’t apply to -alertmanager.storage.path and -alertmanager.storage.retention.\nCortex Alertmanager configuration Cortex Alertmanager can be uploaded via Cortex Set Alertmanager configuration API or using Cortex Tools.\nFollow the instructions at the cortextool link above to download or update to the latest version of the tool.\nTo obtain the full help of how to use cortextool for all commands and flags, use cortextool --help-long.\nThe following example shows the steps to upload the configuration to Cortex Alertmanager using cortextool.\n1. Create the Alertmanager configuration YAML file. The following is amconfig.yml, an example of a configuration for Cortex Alertmanager to send notifications via email:\nglobal: # The smarthost and SMTP sender used for mail notifications. smtp_smarthost: 'localhost:25' smtp_from: 'alertmanager@example.org' smtp_auth_username: 'alertmanager' smtp_auth_password: 'password' route: # A default receiver. receiver: send-email receivers: - name: send-email email_configs: - to: 'someone@localhost' Example on how to set up Slack to support receiving Alertmanager notifications.\n2. Upload the Alertmanager configuration In this example, Cortex Alertmanager is set to be available via localhost on port 8095 with user/org = 100.\nTo upload the above configuration .yml file with --key to be your Basic Authentication or API key:\ncortextool alertmanager load ./amconfig.yml \\ --address=http://localhost:8095 \\ --id=100 \\ --key=\u003cyourKey\u003e If there is no error reported, the upload is successful.\nTo upload the configuration for Cortex Alertmanager using Cortex API and curl - see Cortex Set Alertmanager configuration API.\n3. Ensure the configuration has been uploaded successfully cortextool alertmanager get \\ --address=http://localhost:8095 \\ --id=100 \\ --key=\u003cyourKey\u003e ","categories":"","description":"","excerpt":"Context Cortex Alertmanager notification setup follows mostly the …","ref":"/docs/guides/alertmanager-configuration/","tags":"","title":"Configuring Notification using Cortex Alertmanager"},{"body":"All Cortex components take the tenant ID from a header X-Scope-OrgID on each request. A tenant (also called “user” or “org”) is the owner of a set of series written to and queried from Cortex. All Cortex components trust this value completely: if you need to protect your Cortex installation from accidental or malicious calls, then you must add an additional layer of protection.\nTypically, this means you run Cortex behind a reverse proxy, and you must ensure that all callers, both machines sending data over the remote_write interface and humans sending queries from GUIs, supply credentials which identify them and confirm they are authorised. When configuring the remote_write API in Prometheus, the user and password fields of HTTP Basic auth, or Bearer token, can be used to convey the tenant ID and/or credentials. See the Cortex-Tenant section below for one way to solve this.\nIn trusted environments, Prometheus can send the X-Scope-OrgID header itself by configuring the headers field in its remote_write configuration:\nremote_write: - url: http://\u003ccortex\u003e/prometheus/api/v1/push headers: X-Scope-OrgID: \u003corg\u003e To disable the multi-tenant functionality, you can pass the argument -auth.enabled=false to every Cortex component, which will set the OrgID to the string fake for every request.\nNote that the tenant ID that is used to write the series to the datastore should be the same as the one you use to query the data. If they don’t match, you won’t see any data. As of now, you can’t see series from other tenants.\nFor more information regarding the tenant ID limits, refer to: Tenant ID limitations\nCortex-Tenant One way to add X-Scope-OrgID to Prometheus requests is to use a cortex-tenant proxy which is able to extract the tenant ID from Prometheus labels.\nIt can be placed between Prometheus and Cortex and will search for a predefined label and use its value as X-Scope-OrgID header when proxying the timeseries to Cortex.\nThis can help to run Cortex in a trusted environment where you want to separate your metrics into distinct namespaces by some criteria (e.g. teams, applications, etc.).\nBe advised that cortex-tenant is a third-party community project and it’s not maintained by the Cortex team.\n","categories":"","description":"","excerpt":"All Cortex components take the tenant ID from a header X-Scope-OrgID …","ref":"/docs/guides/auth/","tags":"","title":"Authentication and Authorisation"},{"body":"This doc is likely out of date. It should be updated for blocks storage.\nYou will want to estimate how many nodes are required, how many of each component to run, and how much storage space will be required. In practice, these will vary greatly depending on the metrics being sent to Cortex.\nSome key parameters are:\nThe number of active series. If you have Prometheus already, you can query prometheus_tsdb_head_series to see this number. Sampling rate, e.g. a new sample for each series every minute (the default Prometheus scrape_interval). Multiply this by the number of active series to get the total rate at which samples will arrive at Cortex. The rate at which series are added and removed. This can be very high if you monitor objects that come and go - for example, if you run thousands of batch jobs lasting a minute or so and capture metrics with a unique ID for each one. Read how to analyse this on Prometheus. How compressible the time-series data are. If a metric stays at the same value constantly, then Cortex can compress it very well, so 12 hours of data sampled every 15 seconds would be around 2KB. On the other hand, if the value jumps around a lot, it might take 10KB. There are not currently any tools available to analyse this. How long you want to retain data for, e.g. 1 month or 2 years. Other parameters which can become important if you have particularly high values:\nNumber of different series under one metric name. Number of labels per series. Rate and complexity of queries. Now, some rules of thumb:\nEach million series in an ingester takes 15GB of RAM. The total number of series in ingesters is the number of active series times the replication factor. This is with the default of 12-hour chunks - RAM required will reduce if you set -ingester.max-chunk-age lower (trading off more back-end database I/O). There are some additional considerations for planning for ingester memory usage. Memory increases during write-ahead log (WAL) replay, See Prometheus issue #6934. If you do not have enough memory for WAL replay, the ingester will not be able to restart successfully without intervention. Memory temporarily increases during resharding since timeseries are temporarily on both the new and old ingesters. This means you should scale up the number of ingesters before memory utilization is too high, otherwise you will not have the headroom to account for the temporary increase. Each million series (including churn) consumes 15GB of chunk storage and 4GB of index, per day (so multiply by the retention period). The distributors’ CPU utilization depends on the specific Cortex cluster setup, while they don’t need much RAM. Typically, distributors are capable of processing between 20,000 and 100,000 samples/sec with 1 CPU core. It’s also highly recommended to configure Prometheus max_samples_per_send to 1,000 samples, in order to reduce the distributors’ CPU utilization given the same total samples/sec throughput. If you turn on compression between distributors and ingesters (for example, to save on inter-zone bandwidth charges at AWS/GCP), they will use significantly more CPU (approx. 100% more for distributor and 50% more for ingester).\n","categories":"","description":"","excerpt":"This doc is likely out of date. It should be updated for blocks …","ref":"/docs/guides/capacity-planning/","tags":"","title":"Capacity Planning"},{"body":"Context One option to scale the ruler is by scaling it horizontally. However, with multiple ruler instances running, they will need to coordinate to determine which instance will evaluate which rule. Similar to the ingesters, the rulers establish a hash ring to divide up the responsibilities of evaluating rules.\nConfig In order to enable sharding in the ruler, the following flag needs to be set:\n-ruler.enable-sharding=true In addition, the ruler requires its own ring to be configured, for instance:\n-ruler.ring.consul.hostname=consul.dev.svc.cluster.local:8500 The only configuration that is required is to enable sharding and configure a key-value store. From there, the rulers will shard and handle the division of rules automatically.\nUnlike ingesters, rulers do not hand over responsibility: all rules are re-sharded randomly every time a ruler is added to or removed from the ring.\nRuler Storage The ruler supports six kinds of storage (configdb, Azure, GCS, S3, Swift, local). Most kinds of storage work with the sharded ruler configuration in an obvious way, i.e. configure all rulers to use the same backend.\nThe local implementation reads Prometheus recording rules off the local filesystem. This is a read-only backend that does not support the creation and deletion of rules through the API. Despite the fact that it reads the local filesystem, this method can still be used in a sharded ruler configuration if the operator takes care to load the same rules to every ruler. For instance, this could be accomplished by mounting a Kubernetes ConfigMap onto every ruler pod.\nA typical local config may look something like:\n-ruler-storage.backend=local -ruler-storage.local.directory=/tmp/cortex/rules With the above configuration, the ruler would expect the following layout:\n/tmp/cortex/rules/\u003ctenant id\u003e/rules1.yaml /rules2.yaml Yaml files are expected to be in the Prometheus format.\n","categories":"","description":"","excerpt":"Context One option to scale the ruler is by scaling it horizontally. …","ref":"/docs/guides/ruler-sharding/","tags":"","title":"Config for horizontally scaling the Ruler"},{"body":"Context You can have more than a single Prometheus monitoring and ingesting the same metrics for redundancy. Cortex already does replication for redundancy, and it doesn’t make sense to ingest the same data twice. So in Cortex, we made sure we can dedupe the data we receive from HA Pairs of Prometheus. We do this via the following:\nAssume that there are two teams, each running their own Prometheus, monitoring different services. Let’s call the Prometheus T1 and T2. Now, if the teams are running HA pairs, let’s call the individual Prometheus, T1.a, T1.b, and T2.a and T2.b.\nIn Cortex, we make sure we only ingest from one of T1.a and T1.b, and only from one of T2.a and T2.b. We do this by electing a leader replica for each cluster of Prometheus. For example, in the case of T1, let it be T1.a. As long as T1.a is the leader, we drop the samples sent by T1.b. And if Cortex sees no new samples from T1.a for a short period (30s by default), it’ll switch the leader to be T1.b.\nThis means if T1.a goes down for a few minutes, Cortex’s HA sample handling will have switched and elected T1.b as the leader. This failover timeout is what enables us to only accept samples from a single replica at a time, but ensure we don’t drop too much data in case of issues. Note that with the default scrape period of 15s, and the default timeouts in Cortex, in most cases, you’ll only lose a single scrape of data in the case of a leader election failover. For any rate queries, the rate window should be at least 4x the scrape period to account for any of these failover scenarios, for example, with the default scrape period of 15s, then you should calculate rates over at least 1m periods.\nNow we do the same leader election process for T2.\nConfig Client Side So for Cortex to achieve this, we need 2 identifiers for each process, one identifier for the cluster (T1 or T2, etc.) and one identifier to identify the replica in the cluster (a or b). The easiest way to do this is by setting external labels; the default labels are cluster and __replica__. For example:\ncluster: prom-team1 __replica__: replica1 (or pod-name) and\ncluster: prom-team1 __replica__: replica2 Note: These are external labels and have nothing to do with remote_write config.\nThese two label names are configurable per-tenant within Cortex and should be set to something sensible. For example, the cluster label is already used by some workloads, and you should set the label to be something else that uniquely identifies the cluster. Good examples for this label-name would be team, cluster, prometheus, etc.\nThe replica label should be set so that the value for each prometheus is unique in that cluster. Note: Cortex drops this label when ingesting data but preserves the cluster label. This way, your timeseries won’t change when replicas change.\nServer Side The minimal configuration requires:\nEnabling the HA tracker via -distributor.ha-tracker.enable=true CLI flag (or its YAML config option) Configuring the KV store for the ring (See: Ring/HA Tracker Store). Only Consul and etcd are currently supported. Multi should be used for migration purposes only. Setting the limits configuration to accept samples via -distributor.ha-tracker.enable-for-all-users (or its YAML config option). The following configuration snippet shows an example of the HA tracker config via YAML config file:\nlimits: ... accept_ha_samples: true ... distributor: ... ha_tracker: enable_ha_tracker: true ... kvstore: [store: \u003cstring\u003e | default = \"consul\"] [consul | etcd: \u003cconfig\u003e] ... ... For further configuration file documentation, see the distributor section and Ring/HA Tracker Store.\nFor flag configuration, see the distributor flags having ha-tracker in them.\nRemote Read If you plan to use remote_read, you can’t have the __replica__ label in the external section. Instead, you will need to add it only on the remote_write section of your prometheus.yml.\nglobal: external_labels: cluster: prom-team1 remote_write: - url: https://cortex/api/v1/push write_relabel_configs: - target_label: __replica__ replacement: 1 and\nglobal: external_labels: cluster: prom-team1 remote_write: - url: https://cortex/api/v1/push write_relabel_configs: - target_label: __replica__ replacement: replica2 When Prometheus is executing remote read queries, it will add the external labels to the query. In this case, if it asks for the __replica__ label, Cortex will not return any data.\nTherefore, the __replica__ label should only be added for remote write.\nAccept multiple HA pairs in single request Let’s assume there are two teams (T1 and T2), and each team operates two Prometheus for the HA (T1.a, T1.b for T1 and T2.a, T2.b for T2). They want to operate another Prometheus, receiving whole Prometheus requests and sending write request to the Distributor.\nThe write request flow is as follows: T1.a, T1.b, T2.a, T2.b -\u003e Prometheus -\u003e Distributor which means the Distributor’s incoming write request contains time series of T1.a, T1.b, T2.a, and T2.b. In other words, there are two HA pairs in a single write request, and the expected push result is to accept each Prometheus leader replicas (example: T1.a, T2.b for each team).\nConfig Client side The client setting is the same as a single HA pair. For example:\nFor T1.a\ncluster: prom-team1 __replica__: replica1 (or pod-name) For T1.b\ncluster: prom-team1 __replica__: replica2 (or pod-name) For T2.a\ncluster: prom-team2 __replica__: replica1 (or pod-name) For T2.b\ncluster: prom-team2 __replica__: replica2 (or pod-name) Server side One additional setting is needed to accept multiple HA pairs; it is enabled via --experimental.distributor.ha-tracker.mixed-ha-samples=true (or its YAML config option).\nThe following configuration snippet shows an example of accepting multiple HA pairs config via the YAML config file:\nlimits: ... accept_ha_samples: true accept_mixed_ha_samples: true ... distributor: ... ha_tracker: enable_ha_tracker: true ... kvstore: [ store: \u003cstring\u003e | default = \"consul\" ] [ consul | etcd: \u003cconfig\u003e ] ... ... For further configuration file documentation, see the limits section.\n","categories":"","description":"","excerpt":"Context You can have more than a single Prometheus monitoring and …","ref":"/docs/guides/ha-pair-handling/","tags":"","title":"Config for sending HA Pairs data to Cortex"},{"body":"Welcome! We’re excited that you’re interested in contributing. Below are some basic guidelines.\nWorkflow Cortex follows a standard GitHub pull request workflow. If you’re unfamiliar with this workflow, read the very helpful Understanding the GitHub flow guide from GitHub.\nYou are welcome to create draft PRs at any stage of readiness - this can be helpful to ask for assistance or to develop an idea. But before a piece of work is finished it should:\nBe organized into one or more commits, each of which has a commit message that describes all changes made in that commit (‘why’ more than ‘what’ - we can read the diffs to see the code that changed). Each commit should build towards the whole - don’t leave in back-tracks and mistakes that you later corrected. Have unit and/or integration tests for new functionality or tests that would have caught the bug being fixed. Include a CHANGELOG message if users of Cortex need to hear about what you did. If you have made any changes to flags or config, run make doc and commit the changed files to update the config file documentation. Formatting Cortex projects uses goimports tool (go get golang.org/x/tools/cmd/goimports to install) to format the Go files, and sort imports. We use goimports with -local github.com/cortexproject/cortex parameter, to put Cortex internal imports into a separate group. We try to keep imports sorted into three groups: imports from standard library, imports of 3rd party packages and internal Cortex imports. Goimports will fix the order, but will keep existing newlines between imports in the groups. We try to avoid extra newlines like that.\nYou’re using an IDE you may find useful the following settings for the Cortex project:\nVSCode Developer Certificates of Origin (DCOs) Before submitting your work in a pull request, make sure that all commits are signed off with a Developer Certificate of Origin (DCO). Here’s an example:\ngit commit -s -m \"Here is my signed commit\" You can find further instructions here.\nBuilding Cortex To build:\nmake (By default, the build runs in a Docker container, using an image built with all the tools required. The source code is mounted from where you run make into the build container as a Docker volume.)\nTo run the unit tests suite:\ngo test ./... To run the integration tests suite please see “How integration tests work”.\nDependency management We use Go modules to manage dependencies on external packages. This requires a working Go environment with version 1.11 or greater, git and bzr installed.\nTo add or update a new dependency, use the go get command:\n# Pick the latest tagged release. go get example.com/some/module/pkg # Pick a specific version. go get example.com/some/module/pkg@vX.Y.Z Tidy up the go.mod and go.sum files:\ngo mod tidy go mod vendor git add go.mod go.sum vendor git commit You have to commit the changes to go.mod and go.sum before submitting the pull request.\nDesign patterns and Code conventions Please see the dedicated “Design patterns and Code conventions” page.\nDocumentation The Cortex documentation is compiled into a website published at cortexmetrics.io. Please see “How to run the website locally” for instructions.\nNote: if you attempt to view pages on Github, it’s likely that you might find broken links or pages. That is expected and should not be addressed unless it is causing issues with the site that occur as part of the build.\n","categories":"","description":"","excerpt":"Welcome! We’re excited that you’re interested in contributing. Below …","ref":"/docs/contributing/","tags":"","title":"Contributing"},{"body":"Cortex supports data encryption at rest for some storage backends.\nS3 The Cortex S3 client supports the following server-side encryption (SSE) modes:\nSSE-S3 SSE-KMS Blocks storage The blocks storage S3 server-side encryption can be configured as follows.\ns3_sse_config The s3_sse_config configures the S3 server-side encryption.\nsse: # Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3. # CLI flag: -\u003cprefix\u003e.s3.sse.type [type: \u003cstring\u003e | default = \"\"] # KMS Key ID used to encrypt objects in S3 # CLI flag: -\u003cprefix\u003e.s3.sse.kms-key-id [kms_key_id: \u003cstring\u003e | default = \"\"] # KMS Encryption Context used for object encryption. It expects JSON formatted # string. # CLI flag: -\u003cprefix\u003e.s3.sse.kms-encryption-context [kms_encryption_context: \u003cstring\u003e | default = \"\"] Ruler The ruler S3 server-side encryption can be configured similarly to the blocks storage. The per-tenant overrides are supported when using the storage backend configurable the -ruler-storage. flag prefix (or their respective YAML config options).\nAlertmanager The alertmanager S3 server-side encryption can be configured similarly to the blocks storage. The per-tenant overrides are supported when using the storage backend configurable the -alertmanager-storage. flag prefix (or their respective YAML config options).\nPer-tenant config overrides The S3 client used by the blocks storage, ruler, and alertmanager supports S3 SSE config overrides on a per-tenant basis, using the runtime configuration file. The following settings can be overridden for each tenant:\ns3_sse_type\nS3 server-side encryption type. It must be set to enable the SSE config override for a given tenant. s3_sse_kms_key_id\nS3 server-side encryption KMS Key ID. Ignored if the SSE type override is not set or the type is not SSE-KMS. s3_sse_kms_encryption_context\nS3 server-side encryption KMS encryption context. If unset and the key ID override is set, the encryption context will not be provided to S3. Ignored if the SSE type override is not set or the type is not SSE-KMS. Other storages Other storage backends may support encryption at rest, configuring it directly at the storage level.\n","categories":"","description":"","excerpt":"Cortex supports data encryption at rest for some storage backends.\nS3 …","ref":"/docs/guides/encryption-at-rest/","tags":"","title":"Encryption at Rest"},{"body":"New maintainers are proposed by an existing maintainer and are elected by majority vote. Once the vote passed, the following steps should be done to add a new member to the maintainers team:\nSubmit a PR to add the new member to MAINTAINERS Invite to GitHub organization Invite to cortex-team group Invite to Quay.io repository Invite to Docker Hub organization Submit a PR to CNCF maintainer list for Cortex Invite to CNCF cncf-cortex-maintainers mailing list (via CNCF Service Desk) Invite to Cortex Project Public Calendar Invite to maintainer’s Google Docs Add to the Google Analytics property used for the website statistics Invite to credentials vault ","categories":"","description":"","excerpt":"New maintainers are proposed by an existing maintainer and are elected …","ref":"/docs/contributing/how-to-add-a-maintainer/","tags":"","title":"How to add a maintainer"},{"body":"Cortex ingesters are semi-stateful. A running ingester holds several hours of time series data in memory before they’re flushed to the long-term storage. When an ingester shuts down because of a rolling update or maintenance, the in-memory data must not be discarded in order to avoid any data loss.\nThe Cortex blocks storage requires ingesters to run with a persistent disk where the TSDB WAL and blocks are stored (eg. a StatefulSet when deployed on Kubernetes).\nDuring a rolling update, the leaving ingester closes the open TSDBs, synchronizes the data to disk (fsync), and releases the disk resources. The new ingester, which is expected to reuse the same disk as the leaving one, will replay the TSDB WAL on startup to load back into memory the time series that have not been compacted into a block yet.\n","categories":"","description":"","excerpt":"Cortex ingesters are semi-stateful. A running ingester holds several …","ref":"/docs/guides/ingesters-rolling-updates/","tags":"","title":"Ingesters rolling updates"},{"body":"This guide explains how to scale up and down ingesters.\nIf you’re looking for how to run ingesters rolling updates, please refer to the dedicated guide._\nScaling up Adding more ingesters to a Cortex cluster is considered a safe operation. When a new ingester starts, it will register to the hash ring and the distributors will reshard received series accordingly. Ingesters that were previously receiving those series will see data stop arriving and will consider those series “idle”.\nIf you run with -distributor.shard-by-all-labels=false (the default), before adding a second ingester, you have to wait until data has migrated from idle series to the back-end store; otherwise, you will see gaps in queries. This will happen after the next “head compaction” (typically every 2 hours). If you have set -querier.query-store-after, then that is also a minimum time you have to wait before adding a second ingester.\nIf you run with -distributor.shard-by-all-labels=true, no special care is required to take when scaling up ingesters.\nScaling down A running ingester holds several hours of time series data in memory before they’re flushed to the long-term storage. When an ingester shuts down because of a scale down operation, the in-memory data must not be discarded in order to avoid any data loss.\nIngesters don’t flush series to blocks at shutdown by default. However, Cortex ingesters expose an API endpoint /shutdown that can be called to flush series to blocks and upload blocks to the long-term storage before the ingester terminates.\nEven if ingester blocks are compacted and shipped to the storage at shutdown, it takes some time for queriers and store-gateways to discover the newly uploaded blocks. This is due to the fact that the blocks storage runs a periodic scanning of the storage bucket to discover blocks. If two or more ingesters are scaled down in a short period of time, queriers may miss some data at query time due to series that were stored in the terminated ingesters but their blocks haven’t been discovered yet.\nThe ingesters scale down is deemed an infrequent operation and no automation is currently provided. However, if you need to scale down ingesters, please be aware of the following:\nConfigure queriers and rulers to always query the storage -querier.query-store-after=0s Frequently scan the storage bucket -blocks-storage.bucket-store.sync-interval=5m -compactor.cleanup-interval=5m Lower bucket scanning cache TTLs -blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl=1m -blocks-storage.bucket-store.metadata-cache.tenant-blocks-list-ttl=1m -blocks-storage.bucket-store.metadata-cache.metafile-doesnt-exist-ttl=1m Ingesters should be scaled down one by one: Call /shutdown endpoint on the ingester to shutdown Wait until the HTTP call returns successfully or “finished flushing and shipping TSDB blocks” is logged Terminate the ingester process (the /shutdown will not do it) Before proceeding to the next ingester, wait 2x the maximum between -blocks-storage.bucket-store.sync-interval and -compactor.cleanup-interval ","categories":"","description":"","excerpt":"This guide explains how to scale up and down ingesters.\nIf you’re …","ref":"/docs/guides/ingesters-scaling-up-and-down/","tags":"","title":"Ingesters scaling up and down"},{"body":"Since Cortex is a multi-tenant system, it supports applying limits to each tenant to prevent any single one from using too many resources. In order to help operators understand how close to their limits tenants are, the overrides-exporter module can expose limits as Prometheus metrics.\nContext To update configuration without restarting, Cortex allows operators to supply a runtime_config file that will be periodically reloaded. This file can be specified under the runtime_config section of the main configuration file or using the -runtime-config.file command-line flag. This file is used to apply tenant-specific limits.\nExample The overrides-exporter is not enabled by default; it must be explicitly enabled. We recommend only running a single instance of it in your cluster due to the cardinality of the metrics emitted.\nWith a runtime.yaml file given below:\n# file: runtime.yaml # In this example, we're overriding ingestion limits for a single tenant. overrides: \"user1\": ingestion_burst_size: 350000 ingestion_rate: 350000 max_global_series_per_metric: 300000 max_global_series_per_user: 300000 max_series_per_metric: 0 max_series_per_user: 0 max_samples_per_query: 100000 max_series_per_query: 100000 The overrides-exporter is configured to run as follows:\ncortex -target overrides-exporter -runtime-config.file runtime.yaml -server.http-listen-port=8080 After the overrides-exporter starts, you can use curl to inspect the tenant overrides.\ncurl -s http://localhost:8080/metrics | grep cortex_overrides # HELP cortex_overrides Resource limit overrides applied to tenants # TYPE cortex_overrides gauge cortex_overrides{limit_name=\"ingestion_burst_size\",user=\"user1\"} 350000 cortex_overrides{limit_name=\"ingestion_rate\",user=\"user1\"} 350000 cortex_overrides{limit_name=\"max_global_series_per_metric\",user=\"user1\"} 300000 cortex_overrides{limit_name=\"max_global_series_per_user\",user=\"user1\"} 300000 cortex_overrides{limit_name=\"max_local_series_per_metric\",user=\"user1\"} 0 cortex_overrides{limit_name=\"max_local_series_per_user\",user=\"user1\"} 0 cortex_overrides{limit_name=\"max_samples_per_query\",user=\"user1\"} 100000 cortex_overrides{limit_name=\"max_series_per_query\",user=\"user1\"} 100000 With these metrics, you can set up alerts to know when tenants are close to hitting their limits before they exceed them.\n","categories":"","description":"","excerpt":"Since Cortex is a multi-tenant system, it supports applying limits to …","ref":"/docs/guides/overrides-exporter/","tags":"","title":"Overrides Exporter"},{"body":"This document highlights some ideas for major features we’d like to implement in the near future. To get a more complete overview of planned features and current work, see the issue tracker. Note that these are not ordered by priority.\nHelm charts and other packaging We have a helm chart but it needs work before it can be effectively utilised by different backends. We also don’t provide an official set of dashboards and alerts to our users yet. This is one of the most requested features and something we will tackle in the immediate future. We also plan on publishing debs, rpms along with guides on how to run Cortex on bare-metal.\nAuth Gateway Cortex server has a simple authentication mechanism (X-Scope-OrgId) but users can’t use the multitenancy features out of the box without complicated proxy configuration. It’s hard to support all the different authentication mechanisms used by different companies but plan to have a simple but opinionated auth-gateway that provides value out of the box. The configuration could be as simple as:\ntenants: - name: infra-team password: basic-auth-password - name: api-team password: basic-auth-password2 Billing and Usage analytics We have all the metrics to track how many series, samples and queries each tenant is sending but don’t have dashboards that help with this. We plan to have dashboards and UIs that will help operators monitor and control each tenants usage out of the box.\nDownsampling Downsampling means storing fewer samples, e.g. one per minute instead of one every 15 seconds. This makes queries over long periods more efficient. It can reduce storage space slightly if the full-detail data is discarded.\nPer-metric retention Cortex blocks storage supports deleting all data for a tenant after a time period (e.g. 3 months, 1 year), but we would also like to have custom retention for subsets of metrics (e.g. delete server metrics but retain business metrics).\nExemplar support Exemplars let you link metric samples to other data, such as distributed tracing. As of early 2021 Prometheus will collect exemplars and send them via remote write, but Cortex needs to be extended to handle them.\nScalability Scalability has always been a focus for the project, but there is a lot more work to be done. We can now scale to 100s of Millions of active series but 1 Billion active series is still an unknown.\n","categories":"","description":"","excerpt":"This document highlights some ideas for major features we’d like to …","ref":"/docs/roadmap/","tags":"","title":"Roadmap"},{"body":"Cortex is a distributed system with significant traffic between its services. To allow for secure communication, Cortex supports TLS between all its components. This guide describes the process of setting up TLS.\nGeneration of certs to configure TLS The first step to securing inter-service communication in Cortex with TLS is generating certificates. A Certifying Authority (CA) will be used for this purpose, which should be private to the organization, as any certificates signed by this CA will have permissions to communicate with the cluster.\nWe will use the following script to generate self-signed certs for the cluster:\n# keys openssl genrsa -out root.key openssl genrsa -out client.key openssl genrsa -out server.key # root cert / certifying authority openssl req -x509 -new -nodes -key root.key -subj \"/C=US/ST=KY/O=Org/CN=root\" -sha256 -days 100000 -out root.crt # csrs - certificate signing requests openssl req -new -sha256 -key client.key -subj \"/C=US/ST=KY/O=Org/CN=client\" -out client.csr openssl req -new -sha256 -key server.key -subj \"/C=US/ST=KY/O=Org/CN=localhost\" -out server.csr # certificates openssl x509 -req -in client.csr -CA root.crt -CAkey root.key -CAcreateserial -out client.crt -days 100000 -sha256 openssl x509 -req -in server.csr -CA root.crt -CAkey root.key -CAcreateserial -out server.crt -days 100000 -sha256 Note that the above script generates certificates that are valid for 100,000 days. This can be changed by adjusting the -days option in the above commands. It is recommended that the certs be replaced at least once every 2 years.\nThe above script generates keys client.key, server.key and certs client.crt, server.crt for both the client and server. The CA cert is generated as root.crt.\nLoad certs into the HTTP/GRPC server/client Every HTTP/GRPC link between Cortex components supports TLS configuration through the following config parameters:\nServer flags # Path to the TLS Cert for the HTTP Server -server.http-tls-cert-path=/path/to/server.crt # Path to the TLS Key for the HTTP Server -server.http-tls-key-path=/path/to/server.key # Type of Client Auth for the HTTP Server -server.http-tls-client-auth=\"RequireAndVerifyClientCert\" # Path to the Client CA Cert for the HTTP Server -server.http-tls-ca-path=\"/path/to/root.crt\" # Path to the TLS Cert for the GRPC Server -server.grpc-tls-cert-path=/path/to/server.crt # Path to the TLS Key for the GRPC Server -server.grpc-tls-key-path=/path/to/server.key # Type of Client Auth for the GRPC Server -server.grpc-tls-client-auth=\"RequireAndVerifyClientCert\" # Path to the Client CA Cert for the GRPC Server -server.grpc-tls-ca-path=/path/to/root.crt Client flags Client flags are component-specific.\nFor an HTTP client in the Alertmanager:\n# Path to the TLS Cert for the HTTP Client -alertmanager.configs.tls-cert-path=/path/to/client.crt # Path to the TLS Key for the HTTP Client -alertmanager.configs.tls-key-path=/path/to/client.key # Path to the TLS CA for the HTTP Client -alertmanager.configs.tls-ca-path=/path/to/root.crt For a GRPC client in the Querier:\n# Path to the TLS Cert for the GRPC Client -querier.frontend-client.tls-cert-path=/path/to/client.crt # Path to the TLS Key for the GRPC Client -querier.frontend-client.tls-key-path=/path/to/client.key # Path to the TLS CA for the GRPC Client -querier.frontend-client.tls-ca-path=/path/to/root.crt Similarly, for the GRPC Ingester Client:\n# Path to the TLS Cert for the GRPC Client -ingester.client.tls-cert-path=/path/to/client.crt # Path to the TLS Key for the GRPC Client -ingester.client.tls-key-path=/path/to/client.key # Path to the TLS CA for the GRPC Client -ingester.client.tls-ca-path=/path/to/root.crt TLS can be configured in a similar fashion for other HTTP/GRPC clients in Cortex.\n","categories":"","description":"","excerpt":"Cortex is a distributed system with significant traffic between its …","ref":"/docs/guides/tls/","tags":"","title":"Securing communication between Cortex components with TLS"},{"body":"Cortex must be deployed with due care over system configuration, using principles such as “least privilege” to limit any exposure due to flaws in the source code.\nYou must configure authorisation and authentication externally to Cortex; see this guide\nInformation about security disclosures and mailing lists is in the main repo\n","categories":"","description":"","excerpt":"Cortex must be deployed with due care over system configuration, using …","ref":"/docs/guides/security/","tags":"","title":"Security"},{"body":"Cortex leverages sharding techniques to horizontally scale both single and multi-tenant clusters beyond the capacity of a single node.\nBackground The default sharding strategy employed by Cortex distributes the workload across the entire pool of instances running a given service (eg. ingesters). For example, on the write path, each tenant’s series are sharded across all ingesters, regardless of how many active series the tenant has or how many different tenants are in the cluster.\nThe default strategy allows for a fair balance on the resources consumed by each instance (ie. CPU and memory) and to maximise these resources across the cluster.\nHowever, in a multi-tenant cluster, this approach also introduces some downsides:\nAn outage affects all tenants A misbehaving tenant (eg. causing out-of-memory) could affect all other tenants. The goal of shuffle sharding is to provide an alternative sharding strategy to reduce the blast radius of an outage and better isolate tenants.\nWhat is shuffle sharding? Shuffle sharding is a technique used to isolate different tenants’ workloads and to give each tenant a single-tenant experience even if they’re running in a shared cluster. This technique has been publicly shared and clearly explained by AWS in their builders’ library and a reference implementation has been shown in the Route53 Infima library.\nThe idea is to assign each tenant a shard composed of a subset of the Cortex service instances, aiming to minimize the overlapping instances between two different tenants. Shuffle sharding brings the following benefits over the default sharding strategy:\nAn outage on some Cortex cluster instances/nodes will only affect a subset of tenants. A misbehaving tenant will affect only its shard instances. Due to the low overlap of instances between different tenants, it’s statistically quite likely that any other tenant will run on different instances or only a subset of instances will match the affected ones. Shuffle sharding requires no more resources than the default sharding strategy, but instances may be less evenly balanced from time to time.\nLow overlapping instances probability For example, given a Cortex cluster running 50 ingesters and assigning each tenant 4 out of 50 ingesters, shuffling instances between each tenant, we get 230K possible combinations.\nRandomly picking two different tenants, we have the:\n71% chance that they will not share any instance 26% chance that they will share only 1 instance 2.7% chance that they will share 2 instances 0.08% chance that they will share 3 instances Only a 0.0004% chance that their instances will fully overlap Cortex shuffle sharding Cortex currently supports shuffle sharding in the following services:\nIngesters Query-frontend / Query-scheduler Store-gateway Ruler Compactor Shuffle sharding is disabled by default and needs to be explicitly enabled in the configuration.\nGuaranteed properties The Cortex shuffle sharding implementation guarantees the following properties:\nStability\nGiven a consistent state of the hash ring, the shuffle sharding algorithm always selects the same instances for a given tenant, even across different machines. Consistency\nAdding or removing 1 instance from the hash ring leads to only 1 instance changed at most, in each tenant’s shard. Shuffling\nProbabilistically and for a large enough cluster, it ensures that every tenant gets a different set of instances, with a reduced number of overlapping instances between two tenants to improve failure isolation. Zone-awareness\nWhen zone-aware replication is enabled, the subset of instances selected for each tenant contains a balanced number of instances for each availability zone. Ingesters shuffle sharding By default, the Cortex distributor spreads the received series across all running ingesters.\nWhen shuffle sharding is enabled for the ingesters, the distributor and ruler on the write path spread each tenant series across -distributor.ingestion-tenant-shard-size number of ingesters, while on the read path, the querier and ruler queries only the subset of ingesters holding the series for a given tenant.\nThe shard size can be overridden on a per-tenant basis in the limits overrides configuration.\nIngesters write path To enable shuffle-sharding for ingesters on the write path, you need to configure the following CLI flags (or their respective YAML config options) to distributor, ingester, and ruler:\n-distributor.sharding-strategy=shuffle-sharding -distributor.ingestion-tenant-shard-size=\u003csize\u003e\n\u003csize\u003e should be set to the number of ingesters each tenant series should be sharded to. If \u003csize\u003e is greater than the number of available ingesters in the Cortex cluster, the tenant series are sharded across all ingesters. Ingesters read path Assuming shuffle-sharding has been enabled for the write path, to enable shuffle-sharding for ingesters on the read path too, you need to configure the following CLI flags (or their respective YAML config options) to querier and ruler:\n-distributor.sharding-strategy=shuffle-sharding -distributor.ingestion-tenant-shard-size=\u003csize\u003e -querier.shuffle-sharding-ingesters-lookback-period=\u003cperiod\u003e\nQueriers and rulers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since ’now - lookback period’. The configured lookback \u003cperiod\u003e should be greater than or equal to -querier.query-store-after and -querier.query-ingesters-within if set, and greater than the estimated minimum time it takes for the oldest samples stored in a block uploaded by ingester to be discovered and available for querying (3h with the default configuration). Rollout strategy If you’re running a Cortex cluster with shuffle-sharding disabled and you want to enable it for ingesters, the following rollout strategy should be used to avoid missing querying any time-series in the ingesters’ memory:\nEnable ingesters shuffle-sharding on the write path Wait at least -querier.shuffle-sharding-ingesters-lookback-period time Enable ingesters shuffle-sharding on the read path Limitation: decreasing the tenant shard size The current shuffle-sharding implementation in Cortex has a limitation which prevents safely decreasing the tenant shard size if the ingesters shuffle-sharding is enabled on the read path.\nThe problem is that if a tenant’s subring decreases in size, there is currently no way for the queriers and rulers to know how big the tenant subring was previously, and hence they will potentially miss an ingester with data for that tenant. In other words, the lookback mechanism to select the ingesters which may have received series since ’now - lookback period’ doesn’t work correctly if the tenant shard size is decreased.\nThis is deemed an infrequent operation that we considered banning, but a workaround still exists:\nDisable shuffle-sharding on the read path Decrease the configured tenant shard size Wait at least -querier.shuffle-sharding-ingesters-lookback-period time Re-enable shuffle-sharding on the read path Query-frontend and Query-scheduler shuffle sharding By default, all Cortex queriers can execute received queries for a given tenant.\nWhen shuffle sharding is enabled by setting -frontend.max-queriers-per-tenant (or its respective YAML config option) to a value higher than 0 and lower than the number of available queriers, only the specified number of queriers will execute queries for a single tenant.\nNote that this distribution happens in the query-frontend, or query-scheduler if used. When using query-scheduler, the -frontend.max-queriers-per-tenant option must be set for the query-scheduler component. When not using query-frontend (with or without scheduler), this option is not available.\nThe maximum number of queriers can be overridden on a per-tenant basis in the limits overrides configuration.\nThe impact of “query of death” In the event a tenant is repeatedly sending a “query of death” which leads the querier to crash or get killed because of out-of-memory, the crashed querier will get disconnected from the query-frontend or query-scheduler and a new querier will be immediately assigned to the tenant’s shard. This practically invalidates the assumption that shuffle-sharding can be used to contain the blast radius in case of a query of death.\nTo mitigate it, Cortex allows you to configure a delay between when a querier disconnects because of a crash and when the crashed querier is actually removed from the tenant’s shard (and another healthy querier is added as a replacement). A delay of 1 minute may be a reasonable trade-off:\nQuery-frontend: -query-frontend.querier-forget-delay=1m Query-scheduler: -query-scheduler.querier-forget-delay=1m Store-gateway shuffle sharding The Cortex store-gateway – used by the blocks storage – by default spreads each tenant’s blocks across all running store-gateways.\nWhen shuffle sharding is enabled via -store-gateway.sharding-strategy=shuffle-sharding (or its respective YAML config option), each tenant’s blocks will be sharded across a subset of -store-gateway.tenant-shard-size store-gateway instances. This configuration needs to be set to store-gateway, querier, and ruler.\nThe shard size can be overridden on a per-tenant basis setting store_gateway_tenant_shard_size in the limits overrides configuration.\nPlease check out the store-gateway documentation for more information about how it works.\nRuler shuffle sharding Cortex ruler can run in three modes:\nNo sharding at all. This is the most basic mode of the ruler. It is activated by using -ruler.enable-sharding=false (default) and works correctly only if a single ruler is running. In this mode, the Ruler loads all rules for all tenants. Default sharding, activated by using -ruler.enable-sharding=true and -ruler.sharding-strategy=default (default). In this mode, rulers register themselves into the ring. Each ruler will then select and evaluate only those rules that it “owns”. Shuffle sharding, activated by using -ruler.enable-sharding=true and -ruler.sharding-strategy=shuffle-sharding. Similarly to default sharding, rulers use the ring to distribute workload, but rule groups for each tenant can only be evaluated on a limited number of rulers (-ruler.tenant-shard-size, can also be set per tenant as ruler_tenant_shard_size in overrides). Note that when using sharding strategy, each rule group is evaluated by a single ruler only; there is no replication.\nCompactor shuffle sharding Cortex compactor can run in three modes:\nNo sharding at all. This is the most basic mode of the compactor. It is activated by using -compactor.sharding-enabled=false (default). In this mode, every compactor will run every compaction. Default sharding, activated by using -compactor.sharding-enabled=true and -compactor.sharding-strategy=default (default). In this mode, compactors register themselves into the ring. One single tenant will belong to only 1 compactor. Shuffle sharding, activated by using -compactor.sharding-enabled=true and -compactor.sharding-strategy=shuffle-sharding. Similarly to default sharding, but compactions for each tenant can be carried out on multiple compactors (-compactor.tenant-shard-size, can also be set per tenant as compactor_tenant_shard_size in overrides). With shuffle sharding selected as the sharding strategy, a subset of the compactors will be used to handle a user based on the shard size.\nThe idea behind using the shuffle sharding strategy for the compactor is to further enable horizontal scalability and build tolerance for compactions that may take longer than the compaction interval.\nFAQ Does shuffle sharding add additional overhead to the KV store? No, shuffle sharding subrings are computed client-side and are not stored in the ring. KV store sizing still depends primarily on the number of replicas (of any component that uses the ring, e.g. ingesters) and tokens per replica.\nHowever, each tenant’s subring is cached in memory on the client-side, which may slightly increase the memory footprint of certain components (mostly the distributor).\n","categories":"","description":"","excerpt":"Cortex leverages sharding techniques to horizontally scale both single …","ref":"/docs/guides/shuffle-sharding/","tags":"","title":"Shuffle Sharding"},{"body":"Cortex uses Jaeger or OpenTelemetry to implement distributed tracing. We have found tracing invaluable for troubleshooting the behavior of Cortex in production.\nJaeger Dependencies In order to send traces, you will need to set up a Jaeger deployment. A deployment includes either the Jaeger all-in-one binary or else a distributed system of agents, collectors, and queriers. If running on Kubernetes, Jaeger Kubernetes is an excellent resource.\nConfiguration In order to configure Cortex to send traces, you must do two things:\nSet the JAEGER_AGENT_HOST environment variable in all components to point to your Jaeger agent. This defaults to localhost. Enable sampling in the appropriate components: The Ingester and Ruler self-initiate traces and should have sampling explicitly enabled. Sampling for the Distributor and Query Frontend can be enabled in Cortex or in an upstream service such as your front door. To enable sampling in Cortex components, you can specify either JAEGER_SAMPLER_MANAGER_HOST_PORT for remote sampling or JAEGER_SAMPLER_TYPE and JAEGER_SAMPLER_PARAM to manually set sampling configuration. See the Jaeger Client Go documentation for the full list of environment variables you can configure.\nNote that you must specify one of JAEGER_AGENT_HOST or JAEGER_SAMPLER_MANAGER_HOST_PORT in each component for Jaeger to be enabled, even if you plan to use the default values.\nOpenTelemetry Dependencies In order to send traces, you will need to set up an OpenTelemetry Collector. The collector will be able to send traces to multiple destinations such as AWS X-Ray, Google Cloud, DataDog and [others(https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter). OpenTelemetry Collector provides a helm chart to set up the environment.\nConfiguration See the document on the tracing section in the Configuration file.\nCurrent State Cortex is maintaining backward compatibility with Jaeger support. Cortex has not fully migrated from OpenTracing to OpenTelemetry and is currently using the OpenTracing bridge.\n","categories":"","description":"","excerpt":"Cortex uses Jaeger or OpenTelemetry to implement distributed tracing. …","ref":"/docs/guides/tracing/","tags":"","title":"Tracing"},{"body":"Cortex supports data replication for different services. By default, data is transparently replicated across the whole pool of service instances, regardless of whether these instances are all running within the same availability zone (or data center, or rack) or in different ones.\nIt is completely possible that all the replicas for the given data are held within the same availability zone, even if the Cortex cluster spans multiple zones. Storing multiple replicas for a given data within the same availability zone poses a risk for data loss if there is an outage affecting various nodes within a zone or a full zone outage.\nFor this reason, Cortex optionally supports zone-aware replication. When zone-aware replication is enabled, replicas for the given data are guaranteed to span across different availability zones. This requires the Cortex cluster to run at least in a number of zones equal to the configured replication factor.\nReads from a zone-aware replication enabled Cortex Cluster can withstand zone failures as long as there are no more than floor(replication factor / 2) zones with failing instances.\nThe Cortex services supporting zone-aware replication are:\nDistributors and Ingesters Store-gateways (blocks storage only) Distributors / Ingesters: time-series replication The Cortex time-series replication is used to hold multiple (typically 3) replicas of each time series in the ingesters.\nTo enable the zone-aware replication for the ingesters you should:\nConfigure the availability zone for each ingester via the -ingester.availability-zone CLI flag (or its respective YAML config option) Rollout ingesters to apply the configured zone Enable time-series zone-aware replication via the -distributor.zone-awareness-enabled CLI flag (or its respective YAML config option). Please be aware that this configuration option should be set to distributors, queriers, and rulers. The -distributor.shard-by-all-labels setting has an impact on read availability. When enabled, a metric is sharded across all ingesters, and the querier needs to fetch series from all ingesters. While, when disabled, a metric is sharded only across \u003creplication factor\u003e ingesters.\nIn the event of a large outage impacting ingesters in more than 1 zone, when -distributor.shard-by-all-labels=true, all queries will fail, while when disabled, some queries may still succeed if the ingesters holding the required metric are not impacted by the outage. To learn more about this flag, please refer to distributor arguments.\nStore-gateways: blocks replication The Cortex store-gateway (used only when Cortex is running with the blocks storage) supports blocks sharding, used to horizontally scale blocks in a large cluster without hitting any vertical scalability limit.\nTo enable the zone-aware replication for the store-gateways, please refer to the store-gateway documentation.\nMinimum number of zones For Cortex to function correctly, there must be at least the same number of availability zones as the replication factor. For example, if the replication factor is configured to 3 (default for time-series replication), the Cortex cluster should be spread at least over 3 availability zones.\nIt is safe to have more zones than the replication factor, but it cannot be less. Having fewer availability zones than the replication factor causes a replica write to be missed, and in some cases, the write fails if the availability zones count is too low.\nImpact on unbalanced zones Cortex requires that each zone runs the same number of instances of a given service for which the zone-aware replication is enabled. This guarantees a fair split of the workload across zones.\nOn the contrary, if zones are unbalanced, the zones with a lower number of instances would have a higher pressure on resources utilization (eg. CPU and memory) compared to zones with a higher number of instances.\nImpact on costs Depending on the underlying infrastructure being used, deploying Cortex across multiple availability zones may cause an increase in running costs as most cloud providers charge for inter-availability zone networking. The most significant change would be for a Cortex cluster currently running in a single zone.\n","categories":"","description":"","excerpt":"Cortex supports data replication for different services. By default, …","ref":"/docs/guides/zone-aware-replication/","tags":"","title":"Zone Aware Replication"},{"body":"master / unreleased [CHANGE] OTLP: Change OTLP handler to be consistent with the Prometheus OTLP handler. #6272 target_info metric is enabled by default and can be disabled via -distributor.otlp.disable-target-info=true flag Convert all attributes to labels is disabled by default and can be enabled via -distributor.otlp.convert-all-attributes=true flag You can specify the attributes converted to labels via -distributor.promote-resource-attributes flag. Supported only if -distributor.otlp.convert-all-attributes=false [CHANGE] Change all max async concurrency default values 50 to 3 #6268 [CHANGE] Change default value of -blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency from 50 to 3 #6265 [CHANGE] Enable Compactor and Alertmanager in target all. #6204 [FEATURE] Ruler: Pagination support for List Rules API. #6299 [FEATURE] Query Frontend/Querier: Add protobuf codec -api.querier-default-codec and the option to choose response compression type -querier.response-compression. #5527 [FEATURE] Ruler: Experimental: Add ruler.frontend-address to allow query to query frontends instead of ingesters. #6151 [FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via -ruler.enable-ha-evaluation flag. #6129 [FEATURE] Store Gateway: Add an in-memory chunk cache. #6245 [FEATURE] Chunk Cache: Support multi level cache and add metrics. #6249 [FEATURE] Distributor: Accept multiple HA Tracker pairs in the same request. #6256 [FEATURE] Ruler: Add support for per-user external labels #6340 [ENHANCEMENT] Store Gateway: Add a hedged request to reduce the tail latency. #6388 [ENHANCEMENT] Ingester: Add metrics to track succeed/failed native histograms. #6370 [ENHANCEMENT] Query Frontend/Querier: Add an experimental flag -querier.enable-promql-experimental-functions to enable experimental promQL functions. #6355 [ENHANCEMENT] OTLP: Add -distributor.otlp-max-recv-msg-size flag to limit OTLP request size in bytes. #6333 [ENHANCEMENT] S3 Bucket Client: Add a list objects version configs to configure list api object version. #6280 [ENHANCEMENT] OpenStack Swift: Add application credential configs for Openstack swift object storage backend. #6255 [ENHANCEMENT] Query Frontend: Add new query stats metrics cortex_query_samples_scanned_total and cortex_query_peak_samples to track scannedSamples and peakSample per user. #6228 [ENHANCEMENT] Ingester: Add option ingester.disable-chunk-trimming to disable chunk trimming. #6300 [ENHANCEMENT] Ingester: Add blocks-storage.tsdb.wal-compression-type to support zstd wal compression type. #6232 [ENHANCEMENT] Query Frontend: Add info field to query response. #6207 [ENHANCEMENT] Query Frontend: Add peakSample in query stats response. #6188 [ENHANCEMENT] Ruler: Add new ruler metric cortex_ruler_rule_groups_in_store that is the total rule groups per tenant in store, which can be used to compare with cortex_prometheus_rule_group_rules to count the number of rule groups that are not loaded by a ruler. #5869 [ENHANCEMENT] Ingester/Ring: New READONLY status on ring to be used by Ingester. New ingester API to change mode of ingester #6163 [ENHANCEMENT] Ruler: Add query statistics metrics when –ruler.query-stats-enabled=true. #6173 [ENHANCEMENT] Ingester: Add new API /ingester/all_user_stats which shows loaded blocks, active timeseries and ingestion rate for a specific ingester. #6178 [ENHANCEMENT] Distributor: Add new cortex_reduced_resolution_histogram_samples_total metric to track the number of histogram samples which resolution was reduced. #6182 [ENHANCEMENT] StoreGateway: Implement metadata API limit in queryable. #6195 [ENHANCEMENT] Ingester: Add matchers to ingester LabelNames() and LabelNamesStream() RPC. #6209 [ENHANCEMENT] KV: Add TLS configs to consul. #6374 [ENHANCEMENT] Ingester/Store Gateway Clients: Introduce an experimental HealthCheck handler to quickly fail requests directed to unhealthy targets. #6225 #6257 [ENHANCEMENT] Upgrade build image and Go version to 1.23.2. #6261 #6262 [ENHANCEMENT] Ingester: Introduce a new experimental feature for caching expanded postings on the ingester. #6296 [ENHANCEMENT] Querier/Ruler: Expose store_gateway_consistency_check_max_attempts for max retries when querying store gateway in consistency check. #6276 [ENHANCEMENT] StoreGateway: Add new cortex_bucket_store_chunk_pool_inuse_bytes metric to track the usage in chunk pool. #6310 [ENHANCEMENT] Distributor: Add new cortex_distributor_inflight_client_requests metric to track number of ingester client inflight requests. #6358 [ENHANCEMENT] Distributor: Expose cortex_label_size_bytes native histogram metric. #6372 [ENHANCEMENT] Add new option -server.grpc_server-num-stream-workers to configure the number of worker goroutines that should be used to process incoming streams. #6386 [ENHANCEMENT] Distributor: Return HTTP 5XX instead of HTTP 4XX when instance limits are hit. #6358 [ENHANCEMENT] Ingester: Make sure unregistered ingester joining the ring after WAL replay. #6277 [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224 [BUGFIX] Ruler: Allow rule evaluation to complete during shutdown. #6326 [BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled. #6271 [BUGFIX] Ingester: Fix regression on usage of cortex_ingester_queried_chunks. #6398 [ENHANCEMENT] Distributor: Add a new -distributor.num-push-workers flag to use a goroutine worker pool when sending data from distributor to ingesters. #6406 [BUGFIX] Ingester: Fix possible race condition when active series per LabelSet is configured. #6409 1.18.1 2024-10-14 [BUGFIX] Backporting upgrade to go 1.22.7 to patch CVE-2024-34155, CVE-2024-34156, CVE-2024-34158 #6217 #6264 1.18.0 2024-09-03 [CHANGE] Ingester: Remove -querier.query-store-for-labels-enabled flag. Querying long-term store for labels is always enabled. #5984 [CHANGE] Server: Instrument cortex_request_duration_seconds metric with native histogram. If native-histograms feature is enabled in monitoring Prometheus then the metric name needs to be updated in your dashboards. #6056 [CHANGE] Distributor/Ingester: Change cortex_distributor_ingester_appends_total, cortex_distributor_ingester_append_failures_total, cortex_distributor_ingester_queries_total, and cortex_distributor_ingester_query_failures_total metrics to use the ingester ID instead of its IP as the label value. #6078 [CHANGE] OTLP: Set AddMetricSuffixes to true to always enable metric name normalization. #6136 [CHANGE] Querier: Deprecate and enable by default querier.ingester-metadata-streaming flag. #6147 [CHANGE] QueryFrontend/QueryScheduler: Deprecate -querier.max-outstanding-requests-per-tenant and -query-scheduler.max-outstanding-requests-per-tenant flags. Use frontend.max-outstanding-requests-per-tenant instead. #6146 [CHANGE] Ingesters: Enable ‘snappy-block’ compression on ingester clients by default. #6148 [CHANGE] Ruler: Scheduling ruler.evaluation-delay-duration to be deprecated. Ruler will use the highest value between ruler.evaluation-delay-duration and ruler.query-offset #6149 [CHANGE] Querier: Remove -querier.at-modifier-enabled flag. #6157 [CHANGE] Tracing: Remove deprecated oltp_endpoint config entirely. #6158 [CHANGE] Store Gateway: Enable store gateway zone stable shuffle sharding by default. #6161 [FEATURE] Ingester/Distributor: Experimental: Enable native histogram ingestion via -blocks-storage.tsdb.enable-native-histograms flag. #5986 #6010 #6020 [FEATURE] Querier: Enable querying native histogram chunks. #5944 #6031 [FEATURE] Query Frontend: Support native histogram in query frontend response. #5996 #6043 [FEATURE] Ruler: Support sending native histogram samples to Ingester. #6029 [FEATURE] Ruler: Add support for filtering out alerts in ListRules API. #6011 [FEATURE] Query Frontend: Added a query rejection mechanism to block resource-intensive queries. #6005 [FEATURE] OTLP: Support ingesting OTLP exponential metrics as native histograms. #6071 #6135 [FEATURE] Ingester: Add ingester.instance-limits.max-inflight-query-requests to allow limiting ingester concurrent queries. #6081 [FEATURE] Distributor: Add validation.max-native-histogram-buckets to limit max number of bucket count. Distributor will try to automatically reduce histogram resolution until it is within the bucket limit or resolution cannot be reduced anymore. #6104 [FEATURE] Store Gateway: Introduce token bucket limiter to enhance store gateway throttling. #6016 [FEATURE] Ruler: Add support for query_offset field on RuleGroup and new ruler_query_offset per-tenant limit. #6085 [ENHANCEMENT] Ruler: Add support to persist tokens in rulers. #5987 [ENHANCEMENT] Query Frontend/Querier: Added store gateway postings touched count and touched size in Querier stats and log in Query Frontend. #5892 [ENHANCEMENT] Query Frontend/Querier: Returns warnings on prometheus query responses. #5916 [ENHANCEMENT] Ingester: Allowing to configure -blocks-storage.tsdb.head-compaction-interval flag up to 30 min and add a jitter on the first head compaction. #5919 #5928 [ENHANCEMENT] Distributor: Added max_inflight_push_requests config to ingester client to protect distributor from OOMKilled. #5917 [ENHANCEMENT] Distributor/Querier: Clean stale per-ingester metrics after ingester restarts. #5930 [ENHANCEMENT] Distributor/Ring: Allow disabling detailed ring metrics by ring member. #5931 [ENHANCEMENT] KV: Etcd Added etcd.ping-without-stream-allowed parameter to disable/enable PermitWithoutStream #5933 [ENHANCEMENT] Ingester: Add a new limits_per_label_set limit. This limit functions similarly to max_series_per_metric, but allowing users to define the maximum number of series per LabelSet. #5950 #5993 [ENHANCEMENT] Store Gateway: Log gRPC requests together with headers configured in http_request_headers_to_log. #5958 [ENHANCEMENT] Ingester: Add a new experimental -ingester.labels-string-interning-enabled flag to enable string interning for metrics labels. #6057 [ENHANCEMENT] Ingester: Add link to renew 10% of the ingesters tokens in the admin page. #6063 [ENHANCEMENT] Ruler: Add support for filtering by state and health field on Rules API. #6040 [ENHANCEMENT] Ruler: Add support for filtering by match field on Rules API. #6083 [ENHANCEMENT] Distributor: Reduce memory usage when error volume is high. #6095 [ENHANCEMENT] Compactor: Centralize metrics used by compactor and add user label to compactor metrics. #6096 [ENHANCEMENT] Compactor: Add unique execution ID for each compaction cycle in log for easy debugging. #6097 [ENHANCEMENT] Compactor: Differentiate retry and halt error and retry failed compaction only on retriable error. #6111 [ENHANCEMENT] Ruler: Add support for filtering by state and health field on Rules API. #6040 [ENHANCEMENT] Compactor: Split cleaner cycle for active and deleted tenants. #6112 [ENHANCEMENT] Compactor: Introduce cleaner visit marker. #6113 [ENHANCEMENT] Query Frontend: Add cortex_query_samples_total metric. #6142 [ENHANCEMENT] Ingester: Implement metadata API limit. #6128 [BUGFIX] Configsdb: Fix endline issue in db password. #5920 [BUGFIX] Ingester: Fix user and type labels for the cortex_ingester_tsdb_head_samples_appended_total TSDB metric. #5952 [BUGFIX] Querier: Enforce max query length check for /api/v1/series API even though ignoreMaxQueryLength is set to true. #6018 [BUGFIX] Ingester: Fix issue with the minimize token generator where it was not taking in consideration the current ownership of an instance when generating extra tokens. #6062 [BUGFIX] Scheduler: Fix user queue in scheduler that was not thread-safe. #6077 #6160 [BUGFIX] Ingester: Include out-of-order head compaction when compacting TSDB head. #6108 [BUGFIX] Ingester: Fix cortex_ingester_tsdb_mmap_chunks_total metric. #6134 [BUGFIX] Query Frontend: Fix query rejection bug for metadata queries. #6143 1.17.1 2024-05-20 [CHANGE] Query Frontend/Ruler: Omit empty data, errorType and error fields in API response. #5953 #5954 [ENHANCEMENT] Ingester: Added upload_compacted_blocks_enabled config to ingester to parameterize uploading compacted blocks. #5959 [BUGFIX] Querier: Select correct tenant during query federation. #5943 1.17.0 2024-04-30 [CHANGE] Azure Storage: Upgraded objstore dependency and support Azure Workload Identity Authentication. Added connection_string to support authenticating via SAS token. Marked msi_resource config as deprecating. #5645 [CHANGE] Store Gateway: Add a new fastcache based inmemory index cache. #5619 [CHANGE] Index Cache: Multi level cache backfilling operation becomes async. Added -blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency and -blocks-storage.bucket-store.index-cache.multilevel.max-async-buffer-size configs and metric cortex_store_multilevel_index_cache_backfill_dropped_items_total for number of dropped items. #5661 [CHANGE] Ingester: Disable uploading compacted blocks and overlapping compaction in ingester. #5735 [CHANGE] Distributor: Count the number of rate-limited samples in distributor_samples_in_total. #5714 [CHANGE] Ruler: Remove cortex_ruler_write_requests_total, cortex_ruler_write_requests_failed_total, cortex_ruler_queries_total, cortex_ruler_queries_failed_total, and cortex_ruler_query_seconds_total metrics for the tenant when the ruler deletes the manager for the tenant. #5772 [CHANGE] Main: Mark mem-ballast-size-bytes flag as deprecated. #5816 [CHANGE] Querier: Mark -querier.ingester-streaming flag as deprecated. Now query ingester streaming is always enabled. #5817 [CHANGE] Compactor/Bucket Store: Added -blocks-storage.bucket-store.block-discovery-strategy to configure different block listing strategy. Reverted the current recursive block listing mechanism and use the strategy Concurrent as in 1.15. #5828 [CHANGE] Compactor: Don’t halt compactor when overlapped source blocks detected. #5854 [CHANGE] S3 Bucket Client: Expose -blocks-storage.s3.send-content-md5 flag and set default checksum algorithm to MD5. #5870 [CHANGE] Querier: Mark querier.iterators and querier.batch-iterators flags as deprecated. Now querier always use batch iterators. #5868 [CHANGE] Query Frontend: Error response returned by Query Frontend now follows Prometheus API error response format. #5811 [FEATURE] Experimental: OTLP ingestion. #5813 [FEATURE] Query Frontend/Scheduler: Add query priority support. #5605 [FEATURE] Tracing: Use kuberesolver to resolve OTLP endpoints address with kubernetes:// prefix as Kubernetes service. #5731 [FEATURE] Tracing: Add tracing.otel.round-robin flag to use round_robin gRPC client side LB policy for sending OTLP traces. #5731 [FEATURE] Ruler: Add ruler.concurrent-evals-enabled flag to enable concurrent evaluation within a single rule group for independent rules. Maximum concurrency can be configured via ruler.max-concurrent-evals. #5766 [FEATURE] Distributor Queryable: Experimental: Add config zone_results_quorum_metadata. When querying ingesters using metadata APIs such as label names and values, only results from quorum number of zones will be included and merged. #5779 [FEATURE] Storage Cache Clients: Add config set_async_circuit_breaker_config to utilize the circuit breaker pattern for dynamically thresholding asynchronous set operations. Implemented in both memcached and redis cache clients. #5789 [FEATURE] Ruler: Add experimental experimental.ruler.api-deduplicate-rules flag to remove duplicate rule groups from the Prometheus compatible rules API endpoint. Add experimental ruler.ring.replication-factor and ruler.ring.zone-awareness-enabled flags to configure rule group replication, but only the first ruler in the replicaset evaluates the rule group, the rest will be used as backup to handle events when a ruler is down during an API request to list rules. #5782 #5901 [FEATURE] Ring: Add experimental -ingester.tokens-generator-strategy=minimize-spread flag to enable the new minimize spread token generator strategy. #5855 [FEATURE] Ring Status Page: Add Ownership Diff From Expected column in the ring table to indicate the extent to which the ownership of a specific ingester differs from the expected ownership. #5889 [ENHANCEMENT] Ingester: Add per-tenant new metric cortex_ingester_tsdb_data_replay_duration_seconds. #5477 [ENHANCEMENT] Store Gateway: Added -store-gateway.enabled-tenants and -store-gateway.disabled-tenants to explicitly enable or disable store-gateway for specific tenants. #5638 [ENHANCEMENT] Query Frontend: Write service timing header in response even though there is an error. #5653 [ENHANCEMENT] Compactor: Add new compactor metric cortex_compactor_start_duration_seconds. #5683 [ENHANCEMENT] Index Cache: Multi level cache adds config max_backfill_items to cap max items to backfill per async operation. #5686 [ENHANCEMENT] Query Frontend: Log number of split queries in query stats log. #5703 [ENHANCEMENT] Compactor: Skip compaction retry when encountering a permission denied error. #5727 [ENHANCEMENT] Logging: Added new options for logging HTTP request headers: -server.log-request-headers enables logging HTTP request headers, -server.log-request-headers-exclude-list allows users to specify headers which should not be logged. #5744 [ENHANCEMENT] Query Frontend/Scheduler: Time check in query priority now considers overall data select time window (including range selectors, modifiers and lookback delta). #5758 [ENHANCEMENT] Querier: Added querier.store-gateway-query-stats-enabled to enable or disable store gateway query stats log. #5749 [ENHANCEMENT] Querier: Improve labels APIs latency by merging slices using K-way merge and more than 1 core. #5785 [ENHANCEMENT] AlertManager: Retrying AlertManager Delete Silence on error. #5794 [ENHANCEMENT] Ingester: Add new ingester metric cortex_ingester_max_inflight_query_requests. #5798 [ENHANCEMENT] Query: Added query_storage_wall_time to Query Frontend and Ruler query stats log for wall time spent on fetching data from storage. Query evaluation is not included. #5799 [ENHANCEMENT] Query: Added additional max query length check at Query Frontend and Ruler. Added -querier.ignore-max-query-length flag to disable max query length check at Querier. #5808 [ENHANCEMENT] Querier: Add context error check when converting Metrics to SeriesSet for GetSeries on distributorQuerier. #5827 [ENHANCEMENT] Ruler: Improve GetRules response time by reducing lock contention and introducing a temporary rules cache in ruler/manager.go. #5805 [ENHANCEMENT] Querier: Add context error check when merging slices from ingesters for GetLabel operations. #5837 [BUGFIX] Distributor: Do not use label with empty values for sharding #5717 [BUGFIX] Query Frontend: queries with negative offset should check whether it is cacheable or not. #5719 [BUGFIX] Redis Cache: pass cache_size config correctly. #5734 [BUGFIX] Distributor: Shuffle-Sharding with ingestion_tenant_shard_size set to 0, default sharding strategy should be used. #5189 [BUGFIX] Cortex: Fix GRPC stream clients not honoring overrides for call options. #5797 [BUGFIX] Ruler: Fix support for keep_firing_for field in alert rules. #5823 [BUGFIX] Ring DDB: Fix lifecycle for ring counting unhealthy pods as healthy. #5838 [BUGFIX] Ring DDB: Fix region assignment. #5842 1.16.1 2024-04-23 [ENHANCEMENT] Upgraded Docker base images to alpine:3.18. #5684 [ENHANCEMENT] Upgrade to go 1.21.9 #5879 #5882 1.16.0 2023-11-20 [CHANGE] AlertManager: include reason label in cortex_alertmanager_notifications_failed_total. #5409 [CHANGE] Ruler: Added user label to cortex_ruler_write_requests_total, cortex_ruler_write_requests_failed_total, cortex_ruler_queries_total, and cortex_ruler_queries_failed_total metrics. #5312 [CHANGE] Alertmanager: Validating new fields on the PagerDuty AM config. #5290 [CHANGE] Ingester: Creating label native-histogram-sample on the cortex_discarded_samples_total to keep track of discarded native histogram samples. #5289 [CHANGE] Store Gateway: Rename cortex_bucket_store_cached_postings_compression_time_seconds to cortex_bucket_store_cached_postings_compression_time_seconds_total. #5431 [CHANGE] Store Gateway: Rename cortex_bucket_store_cached_series_fetch_duration_seconds to cortex_bucket_store_series_fetch_duration_seconds and cortex_bucket_store_cached_postings_fetch_duration_seconds to cortex_bucket_store_postings_fetch_duration_seconds. Add new metric cortex_bucket_store_chunks_fetch_duration_seconds. #5448 [CHANGE] Store Gateway: Remove idle_timeout, max_conn_age, pool_size, min_idle_conns fields for Redis index cache and caching bucket. #5448 [CHANGE] Store Gateway: Add flag -store-gateway.sharding-ring.zone-stable-shuffle-sharding to enable store gateway to use zone stable shuffle sharding. #5489 [CHANGE] Bucket Index: Add series_max_size and chunk_max_size to bucket index. #5489 [CHANGE] StoreGateway: Rename cortex_bucket_store_chunk_pool_returned_bytes_total and cortex_bucket_store_chunk_pool_requested_bytes_total to cortex_bucket_store_chunk_pool_operation_bytes_total. #5552 [CHANGE] Query Frontend/Querier: Make build info API disabled by default and add feature flag api.build-info-enabled to enable it. #5533 [CHANGE] Purger: Do no use S3 tenant kms key when uploading deletion marker. #5575 [CHANGE] Ingester: Shipper always allows uploading compacted blocks to ship OOO compacted blocks. #5625 [CHANGE] DDBKV: Change metric name from dynamodb_kv_read_capacity_total to dynamodb_kv_consumed_capacity_total and include Delete, Put, Batch dimension. #5487 [CHANGE] Compactor: Adding the userId on the compact dir path. #5524 [CHANGE] Ingester: Remove deprecated ingester metrics. #5472 [CHANGE] Query Frontend: Expose -querier.max-subquery-steps to configure subquery max steps check. By default, the limit is set to 0, which is disabled. #5656 [FEATURE] Store Gateway: Implementing multi level index cache. #5451 [FEATURE] Ruler: Add support for disabling rule groups. #5521 [FEATURE] Support object storage backends for runtime configuration file. #5292 [FEATURE] Ruler: Add support for Limit field on RuleGroup. #5528 [FEATURE] AlertManager: Add support for Webex, Discord and Telegram Receiver. #5493 [FEATURE] Ingester: added -admin-limit-message to customize the message contained in limit errors.#5460 [FEATURE] AlertManager: Update version to v0.26.0 and bring in Microsoft Teams receiver. #5543 [FEATURE] Store Gateway: Support lazy expanded posting optimization. Added new flag blocks-storage.bucket-store.lazy-expanded-postings-enabled and new metrics cortex_bucket_store_lazy_expanded_postings_total, cortex_bucket_store_lazy_expanded_posting_size_bytes_total and cortex_bucket_store_lazy_expanded_posting_series_overfetched_size_bytes_total. #5556. [FEATURE] Store Gateway: Add max_downloaded_bytes_per_request to limit max bytes to download per store gateway request. #5179 [FEATURE] Added 2 flags -alertmanager.alertmanager-client.grpc-max-send-msg-size and -alertmanager.alertmanager-client.grpc-max-recv-msg-size to configure alert manager grpc client message size limits. #5338 [FEATURE] Querier/StoreGateway: Allow the tenant shard sizes to be a percent of total instances. #5393 [FEATURE] Added the flag -alertmanager.api-concurrency to configure alert manager api concurrency limit. #5412 [FEATURE] Store Gateway: Add -store-gateway.sharding-ring.keep-instance-in-the-ring-on-shutdown to skip unregistering instance from the ring in shutdown. #5421 [FEATURE] Ruler: Support for filtering rules in the API. #5417 [FEATURE] Compactor: Add -compactor.ring.tokens-file-path to store generated tokens locally. #5432 [FEATURE] Query Frontend: Add -frontend.retry-on-too-many-outstanding-requests to re-enqueue 429 requests if there are multiple query-schedulers available. #5496 [FEATURE] Store Gateway: Add -blocks-storage.bucket-store.max-inflight-requests for store gateways to reject further series requests upon reaching the limit. #5553 [FEATURE] Store Gateway: Support filtered index cache. #5587 [ENHANCEMENT] Update go version to 1.21.3. #5630 [ENHANCEMENT] Store Gateway: Add cortex_bucket_store_block_load_duration_seconds histogram to track time to load blocks. #5580 [ENHANCEMENT] Querier: retry chunk pool exhaustion error in querier rather than query frontend. #5569 [ENHANCEMENT] Alertmanager: Added flag -alertmanager.alerts-gc-interval to configure alerts Garbage collection interval. #5550 [ENHANCEMENT] Query Frontend: enable vertical sharding on binary expr . #5507 [ENHANCEMENT] Query Frontend: Include user agent as part of query frontend log. #5450 [ENHANCEMENT] Query: Set CORS Origin headers for Query API #5388 [ENHANCEMENT] Query Frontend: Add cortex_rejected_queries_total metric for throttled queries. #5356 [ENHANCEMENT] Query Frontend: Optimize the decoding of SampleStream. #5349 [ENHANCEMENT] Compactor: Check ctx done when uploading visit marker. #5333 [ENHANCEMENT] AlertManager: Add cortex_alertmanager_dispatcher_aggregation_groups and cortex_alertmanager_dispatcher_alert_processing_duration_seconds metrics for dispatcher. #5592 [ENHANCEMENT] Store Gateway: Added new flag blocks-storage.bucket-store.series-batch-size to control how many series to fetch per batch in Store Gateway. #5582. [ENHANCEMENT] Querier: Log query stats when querying store gateway. #5376 [ENHANCEMENT] Ruler: Add cortex_ruler_rule_group_load_duration_seconds and cortex_ruler_rule_group_sync_duration_seconds metrics. #5609 [ENHANCEMENT] Ruler: Add contextual info and query statistics to log #5604 [ENHANCEMENT] Distributor/Ingester: Add span on push path #5319 [ENHANCEMENT] Query Frontend: Reject subquery with too small step size. #5323 [ENHANCEMENT] Compactor: Exposing Thanos accept-malformed-index to Cortex compactor. #5334 [ENHANCEMENT] Log: Avoid expensive log.Valuer evaluation for disallowed levels. #5297 [ENHANCEMENT] Improving Performance on the API Gzip Handler. #5347 [ENHANCEMENT] Dynamodb: Add puller-sync-time to allow different pull time for ring. #5357 [ENHANCEMENT] Emit querier max_concurrent as a metric. #5362 [ENHANCEMENT] Avoid sort tokens on lifecycler autoJoin. #5394 [ENHANCEMENT] Do not resync blocks in running store gateways during rollout deployment and container restart. #5363 [ENHANCEMENT] Store Gateway: Add new metrics cortex_bucket_store_sent_chunk_size_bytes, cortex_bucket_store_postings_size_bytes and cortex_bucket_store_empty_postings_total. #5397 [ENHANCEMENT] Add jitter to lifecycler heartbeat. #5404 [ENHANCEMENT] Store Gateway: Add config estimated_max_series_size_bytes and estimated_max_chunk_size_bytes to address data overfetch. #5401 [ENHANCEMENT] Distributor/Ingester: Add experimental -distributor.sign_write_requests flag to sign the write requests. #5430 [ENHANCEMENT] Store Gateway/Querier/Compactor: Handling CMK Access Denied errors. #5420 #5442 #5446 [ENHANCEMENT] Alertmanager: Add the alert name in error log when it get throttled. #5456 [ENHANCEMENT] Querier: Retry store gateway on different zones when zone awareness is enabled. #5476 [ENHANCEMENT] Compactor: allow unregister_on_shutdown to be configurable. #5503 [ENHANCEMENT] Querier: Batch adding series to query limiter to optimize locking. #5505 [ENHANCEMENT] Store Gateway: add metric cortex_bucket_store_chunk_refetches_total for number of chunk refetches. #5532 [ENHANCEMENT] BasicLifeCycler: allow final-sleep during shutdown #5517 [ENHANCEMENT] All: Handling CMK Access Denied errors. #5420 #5542 [ENHANCEMENT] Querier: Retry store gateway client connection closing gRPC error. #5558 [ENHANCEMENT] QueryFrontend: Add generic retry for all APIs. #5561. [ENHANCEMENT] Querier: Check context before notifying scheduler and frontend. #5565 [ENHANCEMENT] QueryFrontend: Add metric for number of series requests. #5373 [ENHANCEMENT] Store Gateway: Add histogram metrics for total time spent fetching series and chunks per request. #5573 [ENHANCEMENT] Store Gateway: Check context in multi level cache. Add cortex_store_multilevel_index_cache_fetch_duration_seconds and cortex_store_multilevel_index_cache_backfill_duration_seconds to measure fetch and backfill latency. #5596 [ENHANCEMENT] Ingester: Added new ingester TSDB metrics cortex_ingester_tsdb_head_samples_appended_total, cortex_ingester_tsdb_head_out_of_order_samples_appended_total, cortex_ingester_tsdb_snapshot_replay_error_total, cortex_ingester_tsdb_sample_ooo_delta and cortex_ingester_tsdb_mmap_chunks_total. #5624 [ENHANCEMENT] Query Frontend: Handle context error before decoding and merging responses. #5499 [ENHANCEMENT] Store-Gateway and AlertManager: Add a wait_instance_time_out to context to avoid waiting forever. #5581 [ENHANCEMENT] Blocks storage: Move the tenant deletion mark from \u003ctenantID\u003e/markers/tenant-deletion-mark.json to __markers__/\u003ctenantID\u003e/tenant-deletion-mark.json. #5676 [BUGFIX] Compactor: Fix possible division by zero during compactor config validation. #5535 [BUGFIX] Ruler: Validate if rule group can be safely converted back to rule group yaml from protobuf message #5265 [BUGFIX] Querier: Convert gRPC ResourceExhausted status code from store gateway to 422 limit error. #5286 [BUGFIX] Alertmanager: Route web-ui requests to the alertmanager distributor when sharding is enabled. #5293 [BUGFIX] Storage: Bucket index updater should ignore meta not found for partial blocks. #5343 [BUGFIX] Ring: Add JOINING state to read operation. #5346 [BUGFIX] Compactor: Partial block with only visit marker should be deleted even there is no deletion marker. #5342 [BUGFIX] KV: Etcd calls will no longer block indefinitely and will now time out after the DialTimeout period. #5392 [BUGFIX] Ring: Allow RF greater than number of zones to select more than one instance per zone #5411 [BUGFIX] Store Gateway: Fix bug in store gateway ring comparison logic. #5426 [BUGFIX] Ring: Fix bug in consistency of Get func in a scaling zone-aware ring. #5429 [BUGFIX] Compactor: Fix retry on markers. #5441 [BUGFIX] Query Frontend: Fix bug of failing to cancel downstream request context in query frontend v2 mode (query scheduler enabled). #5447 [BUGFIX] Alertmanager: Remove the user id from state replication key metric label value. #5453 [BUGFIX] Compactor: Avoid cleaner concurrency issues checking global markers before all blocks. #5457 [BUGFIX] DDBKV: Disallow instance with older timestamp to update instance with newer timestamp. #5480 [BUGFIX] DDBKV: When no change detected in ring, retry the CAS until there is change. #5502 [BUGFIX] Fix bug on objstore when configured to use S3 fips endpoints. #5540 [BUGFIX] Ruler: Fix bug on ruler where a failure to load a single RuleGroup would prevent rulers to sync all RuleGroup. #5563 [BUGFIX] Query Frontend: Fix query string being omitted in query stats log. #5655 1.15.3 2023-06-22 [BUGFIX] Distributor: Fix potential data corruption in cases of timeout between distributors and ingesters. #5422 1.15.2 2023-05-09 [ENHANCEMENT] Update Go version to 1.20.4. #5299 1.15.1 2023-04-26 [CHANGE] Alertmanager: Validating new fields on the PagerDuty AM config. #5290 [BUGFIX] Querier: Convert gRPC ResourceExhausted status code from store gateway to 422 limit error. #5286 1.15.0 2023-04-19 [CHANGE] Storage: Make Max exemplars config per tenant instead of global configuration. #5080 #5122 [CHANGE] Alertmanager: Local file disclosure vulnerability in OpsGenie configuration has been fixed. #5045 [CHANGE] Rename oltp_endpoint to otlp_endpoint to match opentelemetry spec and lib name. #5068 [CHANGE] Distributor/Ingester: Log warn level on push requests when they have status code 4xx. Do not log if status is 429. #5103 [CHANGE] Tracing: Use the default OTEL trace sampler when -tracing.otel.exporter-type is set to awsxray. #5141 [CHANGE] Ingester partial error log line to debug level. #5192 [CHANGE] Change HTTP status code from 503/422 to 499 if a request is canceled. #5220 [CHANGE] Store gateways summary metrics have been converted to histograms cortex_bucket_store_series_blocks_queried, cortex_bucket_store_series_data_fetched, cortex_bucket_store_series_data_size_touched_bytes, cortex_bucket_store_series_data_size_fetched_bytes, cortex_bucket_store_series_data_touched, cortex_bucket_store_series_result_series #5239 [FEATURE] Querier/Query Frontend: support Prometheus /api/v1/status/buildinfo API. #4978 [FEATURE] Ingester: Add active series to all_user_stats page. #4972 [FEATURE] Ingester: Added -blocks-storage.tsdb.head-chunks-write-queue-size allowing to configure the size of the in-memory queue used before flushing chunks to the disk . #5000 [FEATURE] Query Frontend: Log query params in query frontend even if error happens. #5005 [FEATURE] Ingester: Enable snapshotting of In-memory TSDB on disk during shutdown via -blocks-storage.tsdb.memory-snapshot-on-shutdown. #5011 [FEATURE] Query Frontend/Scheduler: Add a new counter metric cortex_request_queue_requests_total for total requests going to queue. #5030 [FEATURE] Build ARM docker images. #5041 [FEATURE] Query-frontend/Querier: Create spans to measure time to merge promql responses. #5041 [FEATURE] Querier/Ruler: Support the new thanos promql engine. This is an experimental feature and might change in the future. #5093 [FEATURE] Added zstd as an option for grpc compression #5092 [FEATURE] Ring: Add new kv store option dynamodb. #5026 [FEATURE] Cache: Support redis as backend for caching bucket and index cache. #5057 [FEATURE] Querier/Store-Gateway: Added -blocks-storage.bucket-store.ignore-blocks-within allowing to filter out the recently created blocks from being synced by queriers and store-gateways. #5166 [FEATURE] AlertManager/Ruler: Added support for keep_firing_for on alerting rulers. [FEATURE] Alertmanager: Add support for time_intervals. #5102 [FEATURE] Added snappy-block as an option for grpc compression #5215 [FEATURE] Enable experimental out-of-order samples support. Added 2 new configs ingester.out_of_order_time_window and blocks-storage.tsdb.out_of_order_cap_max. #4964 [ENHANCEMENT] Querier: limit series query to only ingesters if start param is not specified. #4976 [ENHANCEMENT] Query-frontend/scheduler: add a new limit frontend.max-outstanding-requests-per-tenant for configuring queue size per tenant. Started deprecating two flags -query-scheduler.max-outstanding-requests-per-tenant and -querier.max-outstanding-requests-per-tenant, and change their value default to 0. Now if both the old flag and new flag are specified, the old flag’s queue size will be picked. #4991 [ENHANCEMENT] Query-tee: Add /api/v1/query_exemplars API endpoint support. #5010 [ENHANCEMENT] Let blocks_cleaner delete blocks concurrently(default 16 goroutines). #5028 [ENHANCEMENT] Query Frontend/Query Scheduler: Increase upper bound to 60s for queue duration histogram metric. #5029 [ENHANCEMENT] Query Frontend: Log Vertical sharding information when query_stats_enabled is enabled. #5037 [ENHANCEMENT] Ingester: The metadata APIs should honour querier.query-ingesters-within when querier.query-store-for-labels-enabled is true. #5027 [ENHANCEMENT] Query Frontend: Skip instant query roundtripper if sharding is not applicable. #5062 [ENHANCEMENT] Push reduce one hash operation of Labels. #4945 #5114 [ENHANCEMENT] Alertmanager: Added -alertmanager.enabled-tenants and -alertmanager.disabled-tenants to explicitly enable or disable alertmanager for specific tenants. #5116 [ENHANCEMENT] Upgraded Docker base images to alpine:3.17. #5132 [ENHANCEMENT] Add retry logic to S3 bucket client. #5135 [ENHANCEMENT] Update Go version to 1.20.1. #5159 [ENHANCEMENT] Distributor: Reuse byte slices when serializing requests from distributors to ingesters. #5193 [ENHANCEMENT] Query Frontend: Add number of chunks and samples fetched in query stats. #5198 [ENHANCEMENT] Implement grpc.Compressor.DecompressedSize for snappy to optimize memory allocations. #5213 [ENHANCEMENT] Querier: Batch Iterator optimization to prevent transversing it multiple times query ranges steps does not overlap. #5237 [BUGFIX] Updated golang.org/x/net dependency to fix CVE-2022-27664. #5008 [BUGFIX] Fix panic when otel and xray tracing is enabled. #5044 [BUGFIX] Fixed no compact block got grouped in shuffle sharding grouper. #5055 [BUGFIX] Fixed ingesters with less tokens stuck in LEAVING. #5061 [BUGFIX] Tracing: Fix missing object storage span instrumentation. #5074 [BUGFIX] Ingester: Fix Ingesters returning empty response for metadata APIs. #5081 [BUGFIX] Ingester: Fix panic when querying metadata from blocks that are being deleted. #5119 [BUGFIX] Ring: Fix case when dynamodb kv reaches the limit of 25 actions per batch call. #5136 [BUGFIX] Query-frontend: Fix shardable instant queries do not produce sorted results for sort, sort_desc, topk, bottomk functions. #5148, #5170 [BUGFIX] Querier: Fix /api/v1/series returning 5XX instead of 4XX when limits are hit. #5169 [BUGFIX] Compactor: Fix issue that shuffle sharding planner return error if block is under visit by other compactor. #5188 [BUGFIX] Fix S3 BucketWithRetries upload empty content issue #5217 [BUGFIX] Query Frontend: Disable absent, absent_over_time and scalar for vertical sharding. #5221 [BUGFIX] Catch context error in the s3 bucket client. #5240 [BUGFIX] Fix query frontend remote read empty body. #5257 [BUGFIX] Fix query frontend incorrect error response format at SplitByQuery middleware. #5260 1.14.0 2022-12-02 This release removes support for chunks storage. See below for more.\n[CHANGE] Remove support for chunks storage entirely. If you are using chunks storage on a previous version, you must migrate your data on version 1.13.1 or earlier. Before upgrading to this release, you should also remove any deprecated chunks-related configuration, as this release will no longer accept that. The following flags are gone: -dynamodb.* -metrics.* -s3.* -azure.* -bigtable.* -gcs.* -cassandra.* -boltdb.* -local.* some -ingester flags: -ingester.wal-enabled -ingester.checkpoint-enabled -ingester.recover-from-wal -ingester.wal-dir -ingester.checkpoint-duration -ingester.flush-on-shutdown-with-wal-enabled -ingester.max-transfer-retries -ingester.max-samples-per-query -ingester.min-chunk-length -ingester.flush-period -ingester.retain-period -ingester.max-chunk-idle -ingester.max-stale-chunk-idle -ingester.flush-op-timeout -ingester.max-chunk-age -ingester.chunk-age-jitter -ingester.concurrent-flushes -ingester.spread-flushes -ingester.chunk-encoding -store.* except -store.engine and -store.max-query-length -store.query-chunk-limit was deprecated and replaced by -querier.max-fetched-chunks-per-query -deletes.* -grpc-store.* -flusher.wal-dir, -flusher.concurrent-flushes, -flusher.flush-op-timeout [CHANGE] Remove support for alertmanager and ruler legacy store configuration. Before upgrading, you need to convert your configuration to use the alertmanager-storage and ruler-storage configuration on the version that you’re already running, then upgrade. [CHANGE] Disables TSDB isolation. #4825 [CHANGE] Drops support Prometheus 1.x rule format on configdb. #4826 [CHANGE] Removes -ingester.stream-chunks-when-using-blocks experimental flag and stream chunks by default when querier.ingester-streaming is enabled. #4864 [CHANGE] Compactor: Added cortex_compactor_runs_interrupted_total to separate compaction interruptions from failures [CHANGE] Enable PromQL @ modifier, negative offset always. #4927 [CHANGE] Store-gateway: Add user label to cortex_bucket_store_blocks_loaded metric. #4918 [CHANGE] AlertManager: include status label in cortex_alertmanager_alerts_received_total. #4907 [FEATURE] Compactor: Added -compactor.block-files-concurrency allowing to configure number of go routines for download/upload block files during compaction. #4784 [FEATURE] Compactor: Added -compactor.blocks-fetch-concurrency allowing to configure number of go routines for blocks during compaction. #4787 [FEATURE] Compactor: Added configurations for Azure MSI in blocks-storage, ruler-storage and alertmanager-storage. #4818 [FEATURE] Ruler: Add support to pass custom implementations of queryable and pusher. #4782 [FEATURE] Create OpenTelemetry Bridge for Tracing. Now cortex can send traces to multiple destinations using OTEL Collectors. #4834 [FEATURE] Added -api.http-request-headers-to-log allowing for the addition of HTTP Headers to logs #4803 [FEATURE] Distributor: Added a new limit -validation.max-labels-size-bytes allowing to limit the combined size of labels for each timeseries. #4848 [FEATURE] Storage/Bucket: Added -*.s3.bucket-lookup-type allowing to configure the s3 bucket lookup type. #4794 [FEATURE] QueryFrontend: Implement experimental vertical sharding at query frontend for range/instant queries. #4863 [FEATURE] QueryFrontend: Support vertical sharding for subqueries. #4955 [FEATURE] Querier: Added a new limit -querier.max-fetched-data-bytes-per-query allowing to limit the maximum size of all data in bytes that a query can fetch from each ingester and storage. #4854 [FEATURE] Added 2 flags -alertmanager.alertmanager-client.grpc-compression and -querier.store-gateway-client.grpc-compression to configure compression methods for grpc clients. #4889 [ENHANCEMENT] AlertManager: Retrying AlertManager Get Requests (Get Alertmanager status, Get Alertmanager Receivers) on next replica on error #4840 [ENHANCEMENT] Querier/Ruler: Retry store-gateway in case of unexpected failure, instead of failing the query. #4532 #4839 [ENHANCEMENT] Ring: DoBatch prioritize 4xx errors when failing. #4783 [ENHANCEMENT] Cortex now built with Go 1.18. #4829 [ENHANCEMENT] Ingester: Prevent ingesters to become unhealthy during wall replay. #4847 [ENHANCEMENT] Compactor: Introduced visit marker file for blocks so blocks are under compaction will not be picked up by another compactor. #4805 [ENHANCEMENT] Distributor: Add label name to labelValueTooLongError. #4855 [ENHANCEMENT] Enhance traces with hostname information. #4898 [ENHANCEMENT] Improve the documentation around limits. #4905 [ENHANCEMENT] Distributor: cache user overrides to reduce lock contention. #4904 [BUGFIX] Storage/Bucket: Enable AWS SDK for go authentication for s3 to fix IMDSv1 authentication. #4897 [BUGFIX] Memberlist: Add join with no retrying when starting service. #4804 [BUGFIX] Ruler: Fix /ruler/rule_groups returns YAML with extra fields. #4767 [BUGFIX] Respecting -tracing.otel.sample-ratio configuration when enabling OpenTelemetry tracing with X-ray. #4862 [BUGFIX] QueryFrontend: fixed query_range requests when query has start equals to end. #4877 [BUGFIX] AlertManager: fixed issue introduced by #4495 where templates files were being deleted when using alertmanager local store. #4890 [BUGFIX] Ingester: fixed incorrect logging at the start of ingester block shipping logic. #4934 [BUGFIX] Storage/Bucket: fixed global mark missing on deletion. #4949 [BUGFIX] QueryFrontend/Querier: fixed regression added by #4863 where we stopped compressing the response between querier and query frontend. #4960 [BUGFIX] QueryFrontend/Querier: fixed fix response error to be ungzipped when status code is not 2xx. #4975 Known issues Configsdb: Ruler configs doesn’t work. Remove all configs from postgres database that have format Prometheus 1.x rule format before upgrading to v1.14.0 (see 5387) 1.13.0 2022-07-14 [CHANGE] Changed default for -ingester.min-ready-duration from 1 minute to 15 seconds. #4539 [CHANGE] query-frontend: Do not print anything in the logs of query-frontend if a in-progress query has been canceled (context canceled) to avoid spam. #4562 [CHANGE] Compactor block deletion mark migration, needed when upgrading from v1.7, is now disabled by default. #4597 [CHANGE] The status_code label on gRPC client metrics has changed from ‘200’ and ‘500’ to ‘2xx’, ‘5xx’, ‘4xx’, ‘cancel’ or ’error’. #4601 [CHANGE] Memberlist: changed probe interval from 1s to 5s and probe timeout from 500ms to 2s. #4601 [CHANGE] Fix incorrectly named cortex_cache_fetched_keys and cortex_cache_hits metrics. Renamed to cortex_cache_fetched_keys_total and cortex_cache_hits_total respectively. #4686 [CHANGE] Enable Thanos series limiter in store-gateway. #4702 [CHANGE] Distributor: Apply max_fetched_series_per_query limit for /series API. #4683 [CHANGE] Re-enable the proxy_url option for receiver configuration. #4741 [FEATURE] Ruler: Add external_labels option to tag all alerts with a given set of labels. #4499 [FEATURE] Compactor: Add -compactor.skip-blocks-with-out-of-order-chunks-enabled configuration to mark blocks containing index with out-of-order chunks for no compact instead of halting the compaction. #4707 [FEATURE] Querier/Query-Frontend: Add -querier.per-step-stats-enabled and -frontend.cache-queryable-samples-stats configurations to enable query sample statistics. #4708 [FEATURE] Add shuffle sharding for the compactor #4433 [FEATURE] Querier: Use streaming for ingester metadata APIs. #4725 [ENHANCEMENT] Update Go version to 1.17.8. #4602 #4604 #4658 [ENHANCEMENT] Keep track of discarded samples due to bad relabel configuration in cortex_discarded_samples_total. #4503 [ENHANCEMENT] Ruler: Add -ruler.disable-rule-group-label to disable the rule_group label on exported metrics. #4571 [ENHANCEMENT] Query federation: improve performance in MergeQueryable by memoizing labels. #4502 [ENHANCEMENT] Added new ring related config -ingester.readiness-check-ring-health when enabled the readiness probe will succeed only after all instances are ACTIVE and healthy in the ring, this is enabled by default. #4539 [ENHANCEMENT] Added new ring related config -distributor.excluded-zones when set this will exclude the comma-separated zones from the ring, default is “”. #4539 [ENHANCEMENT] Upgraded Docker base images to alpine:3.14. #4514 [ENHANCEMENT] Updated Prometheus to latest. Includes changes from prometheus#9239, adding 15 new functions. Multiple TSDB bugfixes prometheus#9438 \u0026 prometheus#9381. #4524 [ENHANCEMENT] Query Frontend: Add setting -frontend.forward-headers-list in frontend to configure the set of headers from the requests to be forwarded to downstream requests. #4486 [ENHANCEMENT] Blocks storage: Add -blocks-storage.azure.http.*, -alertmanager-storage.azure.http.*, and -ruler-storage.azure.http.* to configure the Azure storage client. #4581 [ENHANCEMENT] Optimise memberlist receive path when used as a backing store for rings with a large number of members. #4601 [ENHANCEMENT] Add length and limit to labelNameTooLongError and labelValueTooLongError #4595 [ENHANCEMENT] Add jitter to rejoinInterval. #4747 [ENHANCEMENT] Compactor: uploading blocks no compaction marks to the global location and introduce a new metric #4729 cortex_bucket_blocks_marked_for_no_compaction_count: Total number of blocks marked for no compaction in the bucket. [ENHANCEMENT] Querier: Reduce the number of series that are kept in memory while streaming from ingesters. #4745 [BUGFIX] AlertManager: remove stale template files. #4495 [BUGFIX] Distributor: fix bug in query-exemplar where some results would get dropped. #4583 [BUGFIX] Update Thanos dependency: compactor tracing support, azure blocks storage memory fix. #4585 [BUGFIX] Set appropriate Content-Type header for /services endpoint, which previously hard-coded text/plain. #4596 [BUGFIX] Querier: Disable query scheduler SRV DNS lookup, which removes noisy log messages about “failed DNS SRV record lookup”. #4601 [BUGFIX] Memberlist: fixed corrupted packets when sending compound messages with more than 255 messages or messages bigger than 64KB. #4601 [BUGFIX] Query Frontend: If ‘LogQueriesLongerThan’ is set to \u003c 0, log all queries as described in the docs. #4633 [BUGFIX] Distributor: update defaultReplicationStrategy to not fail with extend-write when a single instance is unhealthy. #4636 [BUGFIX] Distributor: Fix race condition on /series introduced by #4683. #4716 [BUGFIX] Ruler: Fixed leaking notifiers after users are removed #4718 [BUGFIX] Distributor: Fix a memory leak in distributor due to the cluster label. #4739 [BUGFIX] Memberlist: Avoid clock skew by limiting the timestamp accepted on gossip. #4750 [BUGFIX] Compactor: skip compaction if there is only 1 block available for shuffle-sharding compactor. #4756 [BUGFIX] Compactor: Fixes #4770 - an edge case in compactor with shulffle sharding where compaction stops when a tenant stops ingesting samples. #4771 [BUGFIX] Compactor: fix cortex_compactor_remaining_planned_compactions not set after plan generation for shuffle sharding compactor. #4772 1.11.0 2021-11-25 [CHANGE] Memberlist: Expose default configuration values to the command line options. Note that setting these explicitly to zero will no longer cause the default to be used. If the default is desired, then do set the option. The following are affected: #4276 -memberlist.stream-timeout -memberlist.retransmit-factor -memberlist.pull-push-interval -memberlist.gossip-interval -memberlist.gossip-nodes -memberlist.gossip-to-dead-nodes-time -memberlist.dead-node-reclaim-time [CHANGE] -querier.max-fetched-chunks-per-query previously applied to chunks from ingesters and store separately; now the two combined should not exceed the limit. #4260 [CHANGE] Memberlist: the metric memberlist_kv_store_value_bytes has been removed due to values no longer being stored in-memory as encoded bytes. #4345 [CHANGE] Some files and directories created by Cortex components on local disk now have stricter permissions, and are only readable by owner, but not group or others. #4394 [CHANGE] The metric cortex_deprecated_flags_inuse_total has been renamed to deprecated_flags_inuse_total as part of using grafana/dskit functionality. #4443 [FEATURE] Ruler: Add new -ruler.query-stats-enabled which when enabled will report the cortex_ruler_query_seconds_total as a per-user metric that tracks the sum of the wall time of executing queries in the ruler in seconds. #4317 [FEATURE] Query Frontend: Add cortex_query_fetched_series_total and cortex_query_fetched_chunks_bytes_total per-user counters to expose the number of series and bytes fetched as part of queries. These metrics can be enabled with the -frontend.query-stats-enabled flag (or its respective YAML config option query_stats_enabled). #4343 [FEATURE] AlertManager: Add support for SNS Receiver. #4382 [FEATURE] Distributor: Add label status to metric cortex_distributor_ingester_append_failures_total #4442 [FEATURE] Queries: Added present_over_time PromQL function, also some TSDB optimisations. #4505 [ENHANCEMENT] Add timeout for waiting on compactor to become ACTIVE in the ring. #4262 [ENHANCEMENT] Reduce memory used by streaming queries, particularly in ruler. #4341 [ENHANCEMENT] Ring: allow experimental configuration of disabling of heartbeat timeouts by setting the relevant configuration value to zero. Applies to the following: #4342 -distributor.ring.heartbeat-timeout -ring.heartbeat-timeout -ruler.ring.heartbeat-timeout -alertmanager.sharding-ring.heartbeat-timeout -compactor.ring.heartbeat-timeout -store-gateway.sharding-ring.heartbeat-timeout [ENHANCEMENT] Ring: allow heartbeats to be explicitly disabled by setting the interval to zero. This is considered experimental. This applies to the following configuration options: #4344 -distributor.ring.heartbeat-period -ingester.heartbeat-period -ruler.ring.heartbeat-period -alertmanager.sharding-ring.heartbeat-period -compactor.ring.heartbeat-period -store-gateway.sharding-ring.heartbeat-period [ENHANCEMENT] Memberlist: optimized receive path for processing ring state updates, to help reduce CPU utilization in large clusters. #4345 [ENHANCEMENT] Memberlist: expose configuration of memberlist packet compression via -memberlist.compression=enabled. #4346 [ENHANCEMENT] Update Go version to 1.16.6. #4362 [ENHANCEMENT] Updated Prometheus to include changes from prometheus/prometheus#9083. Now whenever /labels API calls include matchers, blocks store is queried for LabelNames with matchers instead of Series calls which was inefficient. #4380 [ENHANCEMENT] Querier: performance improvements in socket and memory handling. #4429 #4377 [ENHANCEMENT] Exemplars are now emitted for all gRPC calls and many operations tracked by histograms. #4462 [ENHANCEMENT] New options -server.http-listen-network and -server.grpc-listen-network allow binding as ’tcp4’ or ’tcp6’. #4462 [ENHANCEMENT] Rulers: Using shuffle sharding subring on GetRules API. #4466 [ENHANCEMENT] Support memcached auto-discovery via auto-discovery flag, introduced by thanos in https://github.com/thanos-io/thanos/pull/4487. Both AWS and Google Cloud memcached service support auto-discovery, which returns a list of nodes of the memcached cluster. #4412 [BUGFIX] Fixes a panic in the query-tee when comparing result. #4465 [BUGFIX] Frontend: Fixes @ modifier functions (start/end) when splitting queries by time. #4464 [BUGFIX] Compactor: compactor will no longer try to compact blocks that are already marked for deletion. Previously compactor would consider blocks marked for deletion within -compactor.deletion-delay / 2 period as eligible for compaction. #4328 [BUGFIX] HA Tracker: when cleaning up obsolete elected replicas from KV store, tracker didn’t update number of cluster per user correctly. #4336 [BUGFIX] Ruler: fixed counting of PromQL evaluation errors as user-errors when updating cortex_ruler_queries_failed_total. #4335 [BUGFIX] Ingester: When using block storage, prevent any reads or writes while the ingester is stopping. This will prevent accessing TSDB blocks once they have been already closed. #4304 [BUGFIX] Ingester: fixed ingester stuck on start up (LEAVING ring state) when -ingester.heartbeat-period=0 and -ingester.unregister-on-shutdown=false. #4366 [BUGFIX] Ingester: panic during shutdown while fetching batches from cache. #4397 [BUGFIX] Querier: After query-frontend restart, querier may have lower than configured concurrency. #4417 [BUGFIX] Memberlist: forward only changes, not entire original message. #4419 [BUGFIX] Memberlist: don’t accept old tombstones as incoming change, and don’t forward such messages to other gossip members. #4420 [BUGFIX] Querier: fixed panic when querying exemplars and using -distributor.shard-by-all-labels=false. #4473 [BUGFIX] Querier: honor querier minT,maxT if nil SelectHints are passed to Select(). #4413 [BUGFIX] Compactor: fixed panic while collecting Prometheus metrics. #4483 [BUGFIX] Update go-kit package to fix spurious log messages #4544 1.10.0 / 2021-08-03 [CHANGE] Prevent path traversal attack from users able to control the HTTP header X-Scope-OrgID. #4375 (CVE-2021-36157) Users only have control of the HTTP header when Cortex is not frontend by an auth proxy validating the tenant IDs [CHANGE] Enable strict JSON unmarshal for pkg/util/validation.Limits struct. The custom UnmarshalJSON() will now fail if the input has unknown fields. #4298 [CHANGE] Cortex chunks storage has been deprecated and it’s now in maintenance mode: all Cortex users are encouraged to migrate to the blocks storage. No new features will be added to the chunks storage. The default Cortex configuration still runs the chunks engine; please check out the blocks storage doc on how to configure Cortex to run with the blocks storage. #4268 [CHANGE] The example Kubernetes manifests (stored at k8s/) have been removed due to a lack of proper support and maintenance. #4268 [CHANGE] Querier / ruler: deprecated -store.query-chunk-limit CLI flag (and its respective YAML config option max_chunks_per_query) in favour of -querier.max-fetched-chunks-per-query (and its respective YAML config option max_fetched_chunks_per_query). The new limit specifies the maximum number of chunks that can be fetched in a single query from ingesters and long-term storage: the total number of actual fetched chunks could be 2x the limit, being independently applied when querying ingesters and long-term storage. #4125 [CHANGE] Alertmanager: allowed to configure the experimental receivers firewall on a per-tenant basis. The following CLI flags (and their respective YAML config options) have been changed and moved to the limits config section: #4143 -alertmanager.receivers-firewall.block.cidr-networks renamed to -alertmanager.receivers-firewall-block-cidr-networks -alertmanager.receivers-firewall.block.private-addresses renamed to -alertmanager.receivers-firewall-block-private-addresses [CHANGE] Change default value of -server.grpc.keepalive.min-time-between-pings from 5m to 10s and -server.grpc.keepalive.ping-without-stream-allowed to true. #4168 [CHANGE] Ingester: Change default value of -ingester.active-series-metrics-enabled to true. This incurs a small increase in memory usage, between 1.2% and 1.6% as measured on ingesters with 1.3M active series. #4257 [CHANGE] Dependency: update go-redis from v8.2.3 to v8.9.0. #4236 [FEATURE] Querier: Added new -querier.max-fetched-series-per-query flag. When Cortex is running with blocks storage, the max series per query limit is enforced in the querier and applies to unique series received from ingesters and store-gateway (long-term storage). #4179 [FEATURE] Querier/Ruler: Added new -querier.max-fetched-chunk-bytes-per-query flag. When Cortex is running with blocks storage, the max chunk bytes limit is enforced in the querier and ruler and limits the size of all aggregated chunks returned from ingesters and storage as bytes for a query. #4216 [FEATURE] Alertmanager: support negative matchers, time-based muting - upstream release notes. #4237 [FEATURE] Alertmanager: Added rate-limits to notifiers. Rate limits used by all integrations can be configured using -alertmanager.notification-rate-limit, while per-integration rate limits can be specified via -alertmanager.notification-rate-limit-per-integration parameter. Both shared and per-integration limits can be overwritten using overrides mechanism. These limits are applied on individual (per-tenant) alertmanagers. Rate-limited notifications are failed notifications. It is possible to monitor rate-limited notifications via new cortex_alertmanager_notification_rate_limited_total metric. #4135 #4163 [FEATURE] Alertmanager: Added -alertmanager.max-config-size-bytes limit to control size of configuration files that Cortex users can upload to Alertmanager via API. This limit is configurable per-tenant. #4201 [FEATURE] Alertmanager: Added -alertmanager.max-templates-count and -alertmanager.max-template-size-bytes options to control number and size of templates uploaded to Alertmanager via API. These limits are configurable per-tenant. #4223 [FEATURE] Added flag -debug.block-profile-rate to enable goroutine blocking events profiling. #4217 [FEATURE] Alertmanager: The experimental sharding feature is now considered complete. Detailed information about the configuration options can be found here for alertmanager and here for the alertmanager storage. To use the feature: #3925 #4020 #4021 #4031 #4084 #4110 #4126 #4127 #4141 #4146 #4161 #4162 #4222 Ensure that a remote storage backend is configured for Alertmanager to store state using -alertmanager-storage.backend, and flags related to the backend. Note that the local and configdb storage backends are not supported. Ensure that a ring store is configured using -alertmanager.sharding-ring.store, and set the flags relevant to the chosen store type. Enable the feature using -alertmanager.sharding-enabled. Note the prior addition of a new configuration option -alertmanager.persist-interval. This sets the interval between persisting the current alertmanager state (notification log and silences) to object storage. See the configuration file reference for more information. [ENHANCEMENT] Alertmanager: Cleanup persisted state objects from remote storage when a tenant configuration is deleted. #4167 [ENHANCEMENT] Storage: Added the ability to disable Open Census within GCS client (e.g -gcs.enable-opencensus=false). #4219 [ENHANCEMENT] Etcd: Added username and password to etcd config. #4205 [ENHANCEMENT] Alertmanager: introduced new metrics to monitor operation when using -alertmanager.sharding-enabled: #4149 cortex_alertmanager_state_fetch_replica_state_total cortex_alertmanager_state_fetch_replica_state_failed_total cortex_alertmanager_state_initial_sync_total cortex_alertmanager_state_initial_sync_completed_total cortex_alertmanager_state_initial_sync_duration_seconds cortex_alertmanager_state_persist_total cortex_alertmanager_state_persist_failed_total [ENHANCEMENT] Blocks storage: support ingesting exemplars and querying of exemplars. Enabled by setting new CLI flag -blocks-storage.tsdb.max-exemplars=\u003cn\u003e or config option blocks_storage.tsdb.max_exemplars to positive value. #4124 #4181 [ENHANCEMENT] Distributor: Added distributors ring status section in the admin page. #4151 [ENHANCEMENT] Added zone-awareness support to alertmanager for use when sharding is enabled. When zone-awareness is enabled, alerts will be replicated across availability zones. #4204 [ENHANCEMENT] Added tenant_ids tag to tracing spans #4186 [ENHANCEMENT] Ring, query-frontend: Avoid using automatic private IPs (APIPA) when discovering IP address from the interface during the registration of the instance in the ring, or by query-frontend when used with query-scheduler. APIPA still used as last resort with logging indicating usage. #4032 [ENHANCEMENT] Memberlist: introduced new metrics to aid troubleshooting tombstone convergence: #4231 memberlist_client_kv_store_value_tombstones memberlist_client_kv_store_value_tombstones_removed_total memberlist_client_messages_to_broadcast_dropped_total [ENHANCEMENT] Alertmanager: Added -alertmanager.max-dispatcher-aggregation-groups option to control max number of active dispatcher groups in Alertmanager (per tenant, also overridable). When the limit is reached, Dispatcher produces log message and increases cortex_alertmanager_dispatcher_aggregation_group_limit_reached_total metric. #4254 [ENHANCEMENT] Alertmanager: Added -alertmanager.max-alerts-count and -alertmanager.max-alerts-size-bytes to control max number of alerts and total size of alerts that a single user can have in Alertmanager’s memory. Adding more alerts will fail with a log message and incrementing cortex_alertmanager_alerts_insert_limited_total metric (per-user). These limits can be overrided by using per-tenant overrides. Current values are tracked in cortex_alertmanager_alerts_limiter_current_alerts and cortex_alertmanager_alerts_limiter_current_alerts_size_bytes metrics. #4253 [ENHANCEMENT] Store-gateway: added -store-gateway.sharding-ring.wait-stability-min-duration and -store-gateway.sharding-ring.wait-stability-max-duration support to store-gateway, to wait for ring stability at startup. #4271 [ENHANCEMENT] Ruler: added rule_group label to metrics cortex_prometheus_rule_group_iterations_total and cortex_prometheus_rule_group_iterations_missed_total. #4121 [ENHANCEMENT] Ruler: added new metrics for tracking total number of queries and push requests sent to ingester, as well as failed queries and push requests. Failures are only counted for internal errors, but not user-errors like limits or invalid query. This is in contrast to existing cortex_prometheus_rule_evaluation_failures_total, which is incremented also when query or samples appending fails due to user-errors. #4281 cortex_ruler_write_requests_total cortex_ruler_write_requests_failed_total cortex_ruler_queries_total cortex_ruler_queries_failed_total [ENHANCEMENT] Ingester: Added option -ingester.ignore-series-limit-for-metric-names with comma-separated list of metric names that will be ignored in max series per metric limit. #4302 [ENHANCEMENT] Added instrumentation to Redis client, with the following metrics: #3976 cortex_rediscache_request_duration_seconds [BUGFIX] Purger: fix Invalid null value in condition for column range caused by nil value in range for WriteBatch query. #4128 [BUGFIX] Ingester: fixed infrequent panic caused by a race condition between TSDB mmap-ed head chunks truncation and queries. #4176 [BUGFIX] Alertmanager: fix Alertmanager status page if clustering via gossip is disabled or sharding is enabled. #4184 [BUGFIX] Ruler: fix /ruler/rule_groups endpoint doesn’t work when used with object store. #4182 [BUGFIX] Ruler: Honor the evaluation delay for the ALERTS and ALERTS_FOR_STATE series. #4227 [BUGFIX] Make multiple Get requests instead of MGet on Redis Cluster. #4056 [BUGFIX] Ingester: fix issue where runtime limits erroneously override default limits. #4246 [BUGFIX] Ruler: fix startup in single-binary mode when the new ruler_storage is used. #4252 [BUGFIX] Querier: fix queries failing with “at least 1 healthy replica required, could only find 0” error right after scaling up store-gateways until they’re ACTIVE in the ring. #4263 [BUGFIX] Store-gateway: when blocks sharding is enabled, do not load all blocks in each store-gateway in case of a cold startup, but load only blocks owned by the store-gateway replica. #4271 [BUGFIX] Memberlist: fix to setting the default configuration value for -memberlist.retransmit-factor when not provided. This should improve propagation delay of the ring state (including, but not limited to, tombstones). Note that if the configuration is already explicitly given, this fix has no effect. #4269 [BUGFIX] Querier: Fix issue where samples in a chunk might get skipped by batch iterator. #4218 Blocksconvert [ENHANCEMENT] Scanner: add support for DynamoDB (v9 schema only). #3828 [ENHANCEMENT] Add Cassandra support. #3795 [ENHANCEMENT] Scanner: retry failed uploads. #4188 1.9.0 / 2021-05-14 [CHANGE] Alertmanager now removes local files after Alertmanager is no longer running for removed or resharded user. #3910 [CHANGE] Alertmanager now stores local files in per-tenant folders. Files stored by Alertmanager previously are migrated to new hierarchy. Support for this migration will be removed in Cortex 1.11. #3910 [CHANGE] Ruler: deprecated -ruler.storage.* CLI flags (and their respective YAML config options) in favour of -ruler-storage.*. The deprecated config will be removed in Cortex 1.11. #3945 [CHANGE] Alertmanager: deprecated -alertmanager.storage.* CLI flags (and their respective YAML config options) in favour of -alertmanager-storage.*. This change doesn’t apply to alertmanager.storage.path and alertmanager.storage.retention. The deprecated config will be removed in Cortex 1.11. #4002 [CHANGE] Alertmanager: removed -cluster. CLI flags deprecated in Cortex 1.7. The new config options to use are: #3946 -alertmanager.cluster.listen-address instead of -cluster.listen-address -alertmanager.cluster.advertise-address instead of -cluster.advertise-address -alertmanager.cluster.peers instead of -cluster.peer -alertmanager.cluster.peer-timeout instead of -cluster.peer-timeout [CHANGE] Blocks storage: removed the config option -blocks-storage.bucket-store.index-cache.postings-compression-enabled, which was deprecated in Cortex 1.6. Postings compression is always enabled. #4101 [CHANGE] Querier: removed the config option -store.max-look-back-period, which was deprecated in Cortex 1.6 and was used only by the chunks storage. You should use -querier.max-query-lookback instead. #4101 [CHANGE] Query Frontend: removed the config option -querier.compress-http-responses, which was deprecated in Cortex 1.6. You should use-api.response-compression-enabled instead. #4101 [CHANGE] Runtime-config / overrides: removed the config options -limits.per-user-override-config (use -runtime-config.file) and -limits.per-user-override-period (use -runtime-config.reload-period), both deprecated since Cortex 0.6.0. #4112 [CHANGE] Cortex now fails fast on startup if unable to connect to the ring backend. #4068 [FEATURE] The following features have been marked as stable: #4101 Shuffle-sharding Querier support for querying chunks and blocks store at the same time Tracking of active series and exporting them as metrics (-ingester.active-series-metrics-enabled and related flags) Blocks storage: lazy mmap of block indexes in the store-gateway (-blocks-storage.bucket-store.index-header-lazy-loading-enabled) Ingester: close idle TSDB and remove them from local disk (-blocks-storage.tsdb.close-idle-tsdb-timeout) [FEATURE] Memberlist: add TLS configuration options for the memberlist transport layer used by the gossip KV store. #4046 New flags added for memberlist communication: -memberlist.tls-enabled -memberlist.tls-cert-path -memberlist.tls-key-path -memberlist.tls-ca-path -memberlist.tls-server-name -memberlist.tls-insecure-skip-verify [FEATURE] Ruler: added local backend support to the ruler storage configuration under the -ruler-storage. flag prefix. #3932 [ENHANCEMENT] Upgraded Docker base images to alpine:3.13. #4042 [ENHANCEMENT] Blocks storage: reduce ingester memory by eliminating series reference cache. #3951 [ENHANCEMENT] Ruler: optimized \u003cprefix\u003e/api/v1/rules and \u003cprefix\u003e/api/v1/alerts when ruler sharding is enabled. #3916 [ENHANCEMENT] Ruler: added the following metrics when ruler sharding is enabled: #3916 cortex_ruler_clients cortex_ruler_client_request_duration_seconds [ENHANCEMENT] Alertmanager: Add API endpoint to list all tenant alertmanager configs: GET /multitenant_alertmanager/configs. #3529 [ENHANCEMENT] Ruler: Add API endpoint to list all tenant ruler rule groups: GET /ruler/rule_groups. #3529 [ENHANCEMENT] Query-frontend/scheduler: added querier forget delay (-query-frontend.querier-forget-delay and -query-scheduler.querier-forget-delay) to mitigate the blast radius in the event queriers crash because of a repeatedly sent “query of death” when shuffle-sharding is enabled. #3901 [ENHANCEMENT] Query-frontend: reduced memory allocations when serializing query response. #3964 [ENHANCEMENT] Querier / ruler: some optimizations to PromQL query engine. #3934 #3989 [ENHANCEMENT] Ingester: reduce CPU and memory when an high number of errors are returned by the ingester on the write path with the blocks storage. #3969 #3971 #3973 [ENHANCEMENT] Distributor: reduce CPU and memory when an high number of errors are returned by the distributor on the write path. #3990 [ENHANCEMENT] Put metric before label value in the “label value too long” error message. #4018 [ENHANCEMENT] Allow use of y|w|d suffixes for duration related limits and per-tenant limits. #4044 [ENHANCEMENT] Query-frontend: Small optimization on top of PR #3968 to avoid unnecessary Extents merging. #4026 [ENHANCEMENT] Add a metric cortex_compactor_compaction_interval_seconds for the compaction interval config value. #4040 [ENHANCEMENT] Ingester: added following per-ingester (instance) experimental limits: max number of series in memory (-ingester.instance-limits.max-series), max number of users in memory (-ingester.instance-limits.max-tenants), max ingestion rate (-ingester.instance-limits.max-ingestion-rate), and max inflight requests (-ingester.instance-limits.max-inflight-push-requests). These limits are only used when using blocks storage. Limits can also be configured using runtime-config feature, and current values are exported as cortex_ingester_instance_limits metric. #3992. [ENHANCEMENT] Cortex is now built with Go 1.16. #4062 [ENHANCEMENT] Distributor: added per-distributor experimental limits: max number of inflight requests (-distributor.instance-limits.max-inflight-push-requests) and max ingestion rate in samples/sec (-distributor.instance-limits.max-ingestion-rate). If not set, these two are unlimited. Also added metrics to expose current values (cortex_distributor_inflight_push_requests, cortex_distributor_ingestion_rate_samples_per_second) as well as limits (cortex_distributor_instance_limits with various limit label values). #4071 [ENHANCEMENT] Ruler: Added -ruler.enabled-tenants and -ruler.disabled-tenants to explicitly enable or disable rules processing for specific tenants. #4074 [ENHANCEMENT] Block Storage Ingester: /flush now accepts two new parameters: tenant to specify tenant to flush and wait=true to make call synchronous. Multiple tenants can be specified by repeating tenant parameter. If no tenant is specified, all tenants are flushed, as before. #4073 [ENHANCEMENT] Alertmanager: validate configured -alertmanager.web.external-url and fail if ends with /. #4081 [ENHANCEMENT] Alertmanager: added -alertmanager.receivers-firewall.block.cidr-networks and -alertmanager.receivers-firewall.block.private-addresses to block specific network addresses in HTTP-based Alertmanager receiver integrations. #4085 [ENHANCEMENT] Allow configuration of Cassandra’s host selection policy. #4069 [ENHANCEMENT] Store-gateway: retry syncing blocks if a per-tenant sync fails. #3975 #4088 [ENHANCEMENT] Add metric cortex_tcp_connections exposing the current number of accepted TCP connections. #4099 [ENHANCEMENT] Querier: Allow federated queries to run concurrently. #4065 [ENHANCEMENT] Label Values API call now supports match[] parameter when querying blocks on storage (assuming -querier.query-store-for-labels-enabled is enabled). #4133 [BUGFIX] Ruler-API: fix bug where /api/v1/rules/\u003cnamespace\u003e/\u003cgroup_name\u003e endpoint return 400 instead of 404. #4013 [BUGFIX] Distributor: reverted changes done to rate limiting in #3825. #3948 [BUGFIX] Ingester: Fix race condition when opening and closing tsdb concurrently. #3959 [BUGFIX] Querier: streamline tracing spans. #3924 [BUGFIX] Ruler Storage: ignore objects with empty namespace or group in the name. #3999 [BUGFIX] Distributor: fix issue causing distributors to not extend the replication set because of failing instances when zone-aware replication is enabled. #3977 [BUGFIX] Query-frontend: Fix issue where cached entry size keeps increasing when making tiny query repeatedly. #3968 [BUGFIX] Compactor: -compactor.blocks-retention-period now supports weeks (w) and years (y). #4027 [BUGFIX] Querier: returning 422 (instead of 500) when query hits max_chunks_per_query limit with block storage, when the limit is hit in the store-gateway. #3937 [BUGFIX] Ruler: Rule group limit enforcement should now allow the same number of rules in a group as the limit. #3616 [BUGFIX] Frontend, Query-scheduler: allow querier to notify about shutdown without providing any authentication. #4066 [BUGFIX] Querier: fixed race condition causing queries to fail right after querier startup with the “empty ring” error. #4068 [BUGFIX] Compactor: Increment cortex_compactor_runs_failed_total if compactor failed compact a single tenant. #4094 [BUGFIX] Tracing: hot fix to avoid the Jaeger tracing client to indefinitely block the Cortex process shutdown in case the HTTP connection to the tracing backend is blocked. #4134 [BUGFIX] Forward proper EndsAt from ruler to Alertmanager inline with Prometheus behaviour. #4017 [BUGFIX] Querier: support filtering LabelValues with matchers when using tenant federation. #4277 Blocksconvert [ENHANCEMENT] Builder: add -builder.timestamp-tolerance option which may reduce block size by rounding timestamps to make difference whole seconds. #3891 1.8.1 / 2021-04-27 [CHANGE] Fix for CVE-2021-31232: Local file disclosure vulnerability when -experimental.alertmanager.enable-api is used. The HTTP basic auth password_file can be used as an attack vector to send any file content via a webhook. The alertmanager templates can be used as an attack vector to send any file content because the alertmanager can load any text file specified in the templates list. 1.8.0 / 2021-03-24 [CHANGE] Alertmanager: Don’t expose cluster information to tenants via the /alertmanager/api/v1/status API endpoint when operating with clustering enabled. #3903 [CHANGE] Ingester: don’t update internal “last updated” timestamp of TSDB if tenant only sends invalid samples. This affects how “idle” time is computed. #3727 [CHANGE] Require explicit flag -\u003cprefix\u003e.tls-enabled to enable TLS in GRPC clients. Previously it was enough to specify a TLS flag to enable TLS validation. #3156 [CHANGE] Query-frontend: removed -querier.split-queries-by-day (deprecated in Cortex 0.4.0). Please use -querier.split-queries-by-interval instead. #3813 [CHANGE] Store-gateway: the chunks pool controlled by -blocks-storage.bucket-store.max-chunk-pool-bytes is now shared across all tenants. #3830 [CHANGE] Ingester: return error code 400 instead of 429 when per-user/per-tenant series/metadata limits are reached. #3833 [CHANGE] Compactor: add reason label to cortex_compactor_blocks_marked_for_deletion_total metric. Source blocks marked for deletion by compactor are labelled as compaction, while blocks passing the retention period are labelled as retention. #3879 [CHANGE] Alertmanager: the DELETE /api/v1/alerts is now idempotent. No error is returned if the alertmanager config doesn’t exist. #3888 [FEATURE] Experimental Ruler Storage: Add a separate set of configuration options to configure the ruler storage backend under the -ruler-storage. flag prefix. All blocks storage bucket clients and the config service are currently supported. Clients using this implementation will only be enabled if the existing -ruler.storage flags are left unset. #3805 #3864 [FEATURE] Experimental Alertmanager Storage: Add a separate set of configuration options to configure the alertmanager storage backend under the -alertmanager-storage. flag prefix. All blocks storage bucket clients and the config service are currently supported. Clients using this implementation will only be enabled if the existing -alertmanager.storage flags are left unset. #3888 [FEATURE] Adds support to S3 server-side encryption using KMS. The S3 server-side encryption config can be overridden on a per-tenant basis for the blocks storage, ruler and alertmanager. Deprecated -\u003cprefix\u003e.s3.sse-encryption, please use the following CLI flags that have been added. #3651 #3810 #3811 #3870 #3886 #3906 -\u003cprefix\u003e.s3.sse.type -\u003cprefix\u003e.s3.sse.kms-key-id -\u003cprefix\u003e.s3.sse.kms-encryption-context [FEATURE] Querier: Enable @ \u003ctimestamp\u003e modifier in PromQL using the new -querier.at-modifier-enabled flag. #3744 [FEATURE] Overrides Exporter: Add overrides-exporter module for exposing per-tenant resource limit overrides as metrics. It is not included in all target (single-binary mode), and must be explicitly enabled. #3785 [FEATURE] Experimental thanosconvert: introduce an experimental tool thanosconvert to migrate Thanos block metadata to Cortex metadata. #3770 [FEATURE] Alertmanager: It now shards the /api/v1/alerts API using the ring when sharding is enabled. #3671 Added -alertmanager.max-recv-msg-size (defaults to 16M) to limit the size of HTTP request body handled by the alertmanager. New flags added for communication between alertmanagers: -alertmanager.max-recv-msg-size -alertmanager.alertmanager-client.remote-timeout -alertmanager.alertmanager-client.tls-enabled -alertmanager.alertmanager-client.tls-cert-path -alertmanager.alertmanager-client.tls-key-path -alertmanager.alertmanager-client.tls-ca-path -alertmanager.alertmanager-client.tls-server-name -alertmanager.alertmanager-client.tls-insecure-skip-verify [FEATURE] Compactor: added blocks storage per-tenant retention support. This is configured via -compactor.retention-period, and can be overridden on a per-tenant basis. #3879 [ENHANCEMENT] Queries: Instrument queries that were discarded due to the configured max_outstanding_requests_per_tenant. #3894 cortex_query_frontend_discarded_requests_total cortex_query_scheduler_discarded_requests_total [ENHANCEMENT] Ruler: Add TLS and explicit basis authentication configuration options for the HTTP client the ruler uses to communicate with the alertmanager. #3752 -ruler.alertmanager-client.basic-auth-username: Configure the basic authentication username used by the client. Takes precedent over a URL configured username. -ruler.alertmanager-client.basic-auth-password: Configure the basic authentication password used by the client. Takes precedent over a URL configured password. -ruler.alertmanager-client.tls-ca-path: File path to the CA file. -ruler.alertmanager-client.tls-cert-path: File path to the TLS certificate. -ruler.alertmanager-client.tls-insecure-skip-verify: Boolean to disable verifying the certificate. -ruler.alertmanager-client.tls-key-path: File path to the TLS key certificate. -ruler.alertmanager-client.tls-server-name: Expected name on the TLS certificate. [ENHANCEMENT] Ingester: exposed metric cortex_ingester_oldest_unshipped_block_timestamp_seconds, tracking the unix timestamp of the oldest TSDB block not shipped to the storage yet. #3705 [ENHANCEMENT] Prometheus upgraded. #3739 #3806 Avoid unnecessary runtime.GC() during compactions. Prevent compaction loop in TSDB on data gap. [ENHANCEMENT] Query-Frontend now returns server side performance metrics using Server-Timing header when query stats is enabled. #3685 [ENHANCEMENT] Runtime Config: Add a mode query parameter for the runtime config endpoint. /runtime_config?mode=diff now shows the YAML runtime configuration with all values that differ from the defaults. #3700 [ENHANCEMENT] Distributor: Enable downstream projects to wrap distributor push function and access the deserialized write requests before/after they are pushed. #3755 [ENHANCEMENT] Add flag -\u003cprefix\u003e.tls-server-name to require a specific server name instead of the hostname on the certificate. #3156 [ENHANCEMENT] Alertmanager: Remove a tenant’s alertmanager instead of pausing it as we determine it is no longer needed. #3722 [ENHANCEMENT] Blocks storage: added more configuration options to S3 client. #3775 -blocks-storage.s3.tls-handshake-timeout: Maximum time to wait for a TLS handshake. 0 means no limit. -blocks-storage.s3.expect-continue-timeout: The time to wait for a server’s first response headers after fully writing the request headers if the request has an Expect header. 0 to send the request body immediately. -blocks-storage.s3.max-idle-connections: Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit. -blocks-storage.s3.max-idle-connections-per-host: Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. -blocks-storage.s3.max-connections-per-host: Maximum number of connections per host. 0 means no limit. [ENHANCEMENT] Ingester: when tenant’s TSDB is closed, Ingester now removes pushed metrics-metadata from memory, and removes metadata (cortex_ingester_memory_metadata, cortex_ingester_memory_metadata_created_total, cortex_ingester_memory_metadata_removed_total) and validation metrics (cortex_discarded_samples_total, cortex_discarded_metadata_total). #3782 [ENHANCEMENT] Distributor: cleanup metrics for inactive tenants. #3784 [ENHANCEMENT] Ingester: Have ingester to re-emit following TSDB metrics. #3800 cortex_ingester_tsdb_blocks_loaded cortex_ingester_tsdb_reloads_total cortex_ingester_tsdb_reloads_failures_total cortex_ingester_tsdb_symbol_table_size_bytes cortex_ingester_tsdb_storage_blocks_bytes cortex_ingester_tsdb_time_retentions_total [ENHANCEMENT] Querier: distribute workload across -store-gateway.sharding-ring.replication-factor store-gateway replicas when querying blocks and -store-gateway.sharding-enabled=true. #3824 [ENHANCEMENT] Distributor / HA Tracker: added cleanup of unused elected HA replicas from KV store. Added following metrics to monitor this process: #3809 cortex_ha_tracker_replicas_cleanup_started_total cortex_ha_tracker_replicas_cleanup_marked_for_deletion_total cortex_ha_tracker_replicas_cleanup_deleted_total cortex_ha_tracker_replicas_cleanup_delete_failed_total [ENHANCEMENT] Ruler now has new API endpoint /ruler/delete_tenant_config that can be used to delete all ruler groups for tenant. It is intended to be used by administrators who wish to clean up state after removed user. Note that this endpoint is enabled regardless of -experimental.ruler.enable-api. #3750 #3899 [ENHANCEMENT] Query-frontend, query-scheduler: cleanup metrics for inactive tenants. #3826 [ENHANCEMENT] Blocks storage: added -blocks-storage.s3.region support to S3 client configuration. #3811 [ENHANCEMENT] Distributor: Remove cached subrings for inactive users when using shuffle sharding. #3849 [ENHANCEMENT] Store-gateway: Reduced memory used to fetch chunks at query time. #3855 [ENHANCEMENT] Ingester: attempt to prevent idle compaction from happening in concurrent ingesters by introducing a 25% jitter to the configured idle timeout (-blocks-storage.tsdb.head-compaction-idle-timeout). #3850 [ENHANCEMENT] Compactor: cleanup local files for users that are no longer owned by compactor. #3851 [ENHANCEMENT] Store-gateway: close empty bucket stores, and delete leftover local files for tenants that no longer belong to store-gateway. #3853 [ENHANCEMENT] Store-gateway: added metrics to track partitioner behaviour. #3877 cortex_bucket_store_partitioner_requested_bytes_total cortex_bucket_store_partitioner_requested_ranges_total cortex_bucket_store_partitioner_expanded_bytes_total cortex_bucket_store_partitioner_expanded_ranges_total [ENHANCEMENT] Store-gateway: added metrics to monitor chunk buffer pool behaviour. #3880 cortex_bucket_store_chunk_pool_requested_bytes_total cortex_bucket_store_chunk_pool_returned_bytes_total [ENHANCEMENT] Alertmanager: load alertmanager configurations from object storage concurrently, and only load necessary configurations, speeding configuration synchronization process and executing fewer “GET object” operations to the storage when sharding is enabled. #3898 [ENHANCEMENT] Ingester (blocks storage): Ingester can now stream entire chunks instead of individual samples to the querier. At the moment this feature must be explicitly enabled either by using -ingester.stream-chunks-when-using-blocks flag or ingester_stream_chunks_when_using_blocks (boolean) field in runtime config file, but these configuration options are temporary and will be removed when feature is stable. #3889 [ENHANCEMENT] Alertmanager: New endpoint /multitenant_alertmanager/delete_tenant_config to delete configuration for tenant identified by X-Scope-OrgID header. This is an internal endpoint, available even if Alertmanager API is not enabled by using -experimental.alertmanager.enable-api. #3900 [ENHANCEMENT] MemCached: Add max_item_size support. #3929 [BUGFIX] Cortex: Fixed issue where fatal errors and various log messages where not logged. #3778 [BUGFIX] HA Tracker: don’t track as error in the cortex_kv_request_duration_seconds metric a CAS operation intentionally aborted. #3745 [BUGFIX] Querier / ruler: do not log “error removing stale clients” if the ring is empty. #3761 [BUGFIX] Store-gateway: fixed a panic caused by a race condition when the index-header lazy loading is enabled. #3775 #3789 [BUGFIX] Compactor: fixed “could not guess file size” log when uploading blocks deletion marks to the global location. #3807 [BUGFIX] Prevent panic at start if the http_prefix setting doesn’t have a valid value. #3796 [BUGFIX] Memberlist: fixed panic caused by race condition in armon/go-metrics used by memberlist client. #3725 [BUGFIX] Querier: returning 422 (instead of 500) when query hits max_chunks_per_query limit with block storage. #3895 [BUGFIX] Alertmanager: Ensure that experimental /api/v1/alerts endpoints work when -http.prefix is empty. #3905 [BUGFIX] Chunk store: fix panic in inverted index when deleted fingerprint is no longer in the index. #3543 1.7.1 / 2021-04-27 [CHANGE] Fix for CVE-2021-31232: Local file disclosure vulnerability when -experimental.alertmanager.enable-api is used. The HTTP basic auth password_file can be used as an attack vector to send any file content via a webhook. The alertmanager templates can be used as an attack vector to send any file content because the alertmanager can load any text file specified in the templates list. 1.7.0 / 2021-02-23 Note the blocks storage compactor runs a migration task at startup in this version, which can take many minutes and use a lot of RAM. Turn this off after first run.\n[CHANGE] FramedSnappy encoding support has been removed from Push and Remote Read APIs. This means Prometheus 1.6 support has been removed and the oldest Prometheus version supported in the remote write is 1.7. #3682 [CHANGE] Ruler: removed the flag -ruler.evaluation-delay-duration-deprecated which was deprecated in 1.4.0. Please use the ruler_evaluation_delay_duration per-tenant limit instead. #3694 [CHANGE] Removed the flags -\u003cprefix\u003e.grpc-use-gzip-compression which were deprecated in 1.3.0: #3694 -query-scheduler.grpc-client-config.grpc-use-gzip-compression: use -query-scheduler.grpc-client-config.grpc-compression instead -frontend.grpc-client-config.grpc-use-gzip-compression: use -frontend.grpc-client-config.grpc-compression instead -ruler.client.grpc-use-gzip-compression: use -ruler.client.grpc-compression instead -bigtable.grpc-use-gzip-compression: use -bigtable.grpc-compression instead -ingester.client.grpc-use-gzip-compression: use -ingester.client.grpc-compression instead -querier.frontend-client.grpc-use-gzip-compression: use -querier.frontend-client.grpc-compression instead [CHANGE] Querier: it’s not required to set -frontend.query-stats-enabled=true in the querier anymore to enable query statistics logging in the query-frontend. The flag is now required to be configured only in the query-frontend and it will be propagated to the queriers. #3595 #3695 [CHANGE] Blocks storage: compactor is now required when running a Cortex cluster with the blocks storage, because it also keeps the bucket index updated. #3583 [CHANGE] Blocks storage: block deletion marks are now stored in a per-tenant global markers/ location too, other than within the block location. The compactor, at startup, will copy deletion marks from the block location to the global location. This migration is required only once, so it can be safely disabled via -compactor.block-deletion-marks-migration-enabled=false after new compactor has successfully started at least once in the cluster. #3583 [CHANGE] OpenStack Swift: the default value for the -ruler.storage.swift.container-name and -swift.container-name config options has changed from cortex to empty string. If you were relying on the default value, please set it back to cortex. #3660 [CHANGE] HA Tracker: configured replica label is now verified against label value length limit (-validation.max-length-label-value). #3668 [CHANGE] Distributor: extend_writes field in YAML configuration has moved from lifecycler (inside ingester_config) to distributor_config. This doesn’t affect command line option -distributor.extend-writes, which stays the same. #3719 [CHANGE] Alertmanager: Deprecated -cluster. CLI flags in favor of their -alertmanager.cluster. equivalent. The deprecated flags (and their respective YAML config options) are: #3677 -cluster.listen-address in favor of -alertmanager.cluster.listen-address -cluster.advertise-address in favor of -alertmanager.cluster.advertise-address -cluster.peer in favor of -alertmanager.cluster.peers -cluster.peer-timeout in favor of -alertmanager.cluster.peer-timeout [CHANGE] Blocks storage: the default value of -blocks-storage.bucket-store.sync-interval has been changed from 5m to 15m. #3724 [FEATURE] Querier: Queries can be federated across multiple tenants. The tenants IDs involved need to be specified separated by a | character in the X-Scope-OrgID request header. This is an experimental feature, which can be enabled by setting -tenant-federation.enabled=true on all Cortex services. #3250 [FEATURE] Alertmanager: introduced the experimental option -alertmanager.sharding-enabled to shard tenants across multiple Alertmanager instances. This feature is still under heavy development and its usage is discouraged. The following new metrics are exported by the Alertmanager: #3664 cortex_alertmanager_ring_check_errors_total cortex_alertmanager_sync_configs_total cortex_alertmanager_sync_configs_failed_total cortex_alertmanager_tenants_discovered cortex_alertmanager_tenants_owned [ENHANCEMENT] Allow specifying JAEGER_ENDPOINT instead of sampling server or local agent port. #3682 [ENHANCEMENT] Blocks storage: introduced a per-tenant bucket index, periodically updated by the compactor, used to avoid full bucket scanning done by queriers, store-gateways and rulers. The bucket index is updated by the compactor during blocks cleanup, on every -compactor.cleanup-interval. #3553 #3555 #3561 #3583 #3625 #3711 #3715 [ENHANCEMENT] Blocks storage: introduced an option -blocks-storage.bucket-store.bucket-index.enabled to enable the usage of the bucket index in the querier, store-gateway and ruler. When enabled, the querier, store-gateway and ruler will use the bucket index to find a tenant’s blocks instead of running the periodic bucket scan. The following new metrics are exported by the querier and ruler: #3614 #3625 cortex_bucket_index_loads_total cortex_bucket_index_load_failures_total cortex_bucket_index_load_duration_seconds cortex_bucket_index_loaded [ENHANCEMENT] Compactor: exported the following metrics. #3583 #3625 cortex_bucket_blocks_count: Total number of blocks per tenant in the bucket. Includes blocks marked for deletion, but not partial blocks. cortex_bucket_blocks_marked_for_deletion_count: Total number of blocks per tenant marked for deletion in the bucket. cortex_bucket_blocks_partials_count: Total number of partial blocks. cortex_bucket_index_last_successful_update_timestamp_seconds: Timestamp of the last successful update of a tenant’s bucket index. [ENHANCEMENT] Ruler: Add cortex_prometheus_last_evaluation_samples to expose the number of samples generated by a rule group per tenant. #3582 [ENHANCEMENT] Memberlist: add status page (/memberlist) with available details about memberlist-based KV store and memberlist cluster. It’s also possible to view KV values in Go struct or JSON format, or download for inspection. #3575 [ENHANCEMENT] Memberlist: client can now keep a size-bounded buffer with sent and received messages and display them in the admin UI (/memberlist) for troubleshooting. #3581 #3602 [ENHANCEMENT] Blocks storage: added block index attributes caching support to metadata cache. The TTL can be configured via -blocks-storage.bucket-store.metadata-cache.block-index-attributes-ttl. #3629 [ENHANCEMENT] Alertmanager: Add support for Azure blob storage. #3634 [ENHANCEMENT] Compactor: tenants marked for deletion will now be fully cleaned up after some delay since deletion of last block. Cleanup includes removal of remaining marker files (including tenant deletion mark file) and files under debug/metas. #3613 [ENHANCEMENT] Compactor: retry compaction of a single tenant on failure instead of re-running compaction for all tenants. #3627 [ENHANCEMENT] Querier: Implement result caching for tenant query federation. #3640 [ENHANCEMENT] API: Add a mode query parameter for the config endpoint: #3645 /config?mode=diff: Shows the YAML configuration with all values that differ from the defaults. /config?mode=defaults: Shows the YAML configuration with all the default values. [ENHANCEMENT] OpenStack Swift: added the following config options to OpenStack Swift backend client: #3660 Chunks storage: -swift.auth-version, -swift.max-retries, -swift.connect-timeout, -swift.request-timeout. Blocks storage: -blocks-storage.swift.auth-version, -blocks-storage.swift.max-retries, -blocks-storage.swift.connect-timeout, -blocks-storage.swift.request-timeout. Ruler: -ruler.storage.swift.auth-version, -ruler.storage.swift.max-retries, -ruler.storage.swift.connect-timeout, -ruler.storage.swift.request-timeout. [ENHANCEMENT] Disabled in-memory shuffle-sharding subring cache in the store-gateway, ruler and compactor. This should reduce the memory utilisation in these services when shuffle-sharding is enabled, without introducing a significantly increase CPU utilisation. #3601 [ENHANCEMENT] Shuffle sharding: optimised subring generation used by shuffle sharding. #3601 [ENHANCEMENT] New /runtime_config endpoint that returns the defined runtime configuration in YAML format. The returned configuration includes overrides. #3639 [ENHANCEMENT] Query-frontend: included the parameter name failed to validate in HTTP 400 message. #3703 [ENHANCEMENT] Fail to startup Cortex if provided runtime config is invalid. #3707 [ENHANCEMENT] Alertmanager: Add flags to customize the cluster configuration: #3667 -alertmanager.cluster.gossip-interval: The interval between sending gossip messages. By lowering this value (more frequent) gossip messages are propagated across cluster more quickly at the expense of increased bandwidth usage. -alertmanager.cluster.push-pull-interval: The interval between gossip state syncs. Setting this interval lower (more frequent) will increase convergence speeds across larger clusters at the expense of increased bandwidth usage. [ENHANCEMENT] Distributor: change the error message returned when a received series has too many label values. The new message format has the series at the end and this plays better with Prometheus logs truncation. #3718 From: sample for '\u003cseries\u003e' has \u003cvalue\u003e label names; limit \u003cvalue\u003e To: series has too many labels (actual: \u003cvalue\u003e, limit: \u003cvalue\u003e) series: '\u003cseries\u003e' [ENHANCEMENT] Improve bucket index loader to handle edge case where new tenant has not had blocks uploaded to storage yet. #3717 [BUGFIX] Allow -querier.max-query-lookback use y|w|d suffix like deprecated -store.max-look-back-period. #3598 [BUGFIX] Memberlist: Entry in the ring should now not appear again after using “Forget” feature (unless it’s still heartbeating). #3603 [BUGFIX] Ingester: do not close idle TSDBs while blocks shipping is in progress. #3630 #3632 [BUGFIX] Ingester: correctly update cortex_ingester_memory_users and cortex_ingester_active_series when a tenant’s idle TSDB is closed, when running Cortex with the blocks storage. #3646 [BUGFIX] Querier: fix default value incorrectly overriding -querier.frontend-address in single-binary mode. #3650 [BUGFIX] Compactor: delete deletion-mark.json at last when deleting a block in order to not leave partial blocks without deletion mark in the bucket if the compactor is interrupted while deleting a block. #3660 [BUGFIX] Blocks storage: do not cleanup a partially uploaded block when meta.json upload fails. Despite failure to upload meta.json, this file may in some cases still appear in the bucket later. By skipping early cleanup, we avoid having corrupted blocks in the storage. #3660 [BUGFIX] Alertmanager: disable access to /alertmanager/metrics (which exposes all Cortex metrics), /alertmanager/-/reload and /alertmanager/debug/*, which were available to any authenticated user with enabled AlertManager. #3678 [BUGFIX] Query-Frontend: avoid creating many small sub-queries by discarding cache extents under 5 minutes #3653 [BUGFIX] Ruler: Ensure the stale markers generated for evaluated rules respect the configured -ruler.evaluation-delay-duration. This will avoid issues with samples with NaN be persisted with timestamps set ahead of the next rule evaluation. #3687 [BUGFIX] Alertmanager: don’t serve HTTP requests until Alertmanager has fully started. Serving HTTP requests earlier may result in loss of configuration for the user. #3679 [BUGFIX] Do not log “failed to load config” if runtime config file is empty. #3706 [BUGFIX] Do not allow to use a runtime config file containing multiple YAML documents. #3706 [BUGFIX] HA Tracker: don’t track as error in the cortex_kv_request_duration_seconds metric a CAS operation intentionally aborted. #3745 1.6.0 / 2020-12-29 [CHANGE] Query Frontend: deprecate -querier.compress-http-responses in favour of -api.response-compression-enabled. #3544 [CHANGE] Querier: deprecated -store.max-look-back-period. You should use -querier.max-query-lookback instead. #3452 [CHANGE] Blocks storage: increased -blocks-storage.bucket-store.chunks-cache.attributes-ttl default from 24h to 168h (1 week). #3528 [CHANGE] Blocks storage: the config option -blocks-storage.bucket-store.index-cache.postings-compression-enabled has been deprecated and postings compression is always enabled. #3538 [CHANGE] Ruler: gRPC message size default limits on the Ruler-client side have changed: #3523 limit for outgoing gRPC messages has changed from 2147483647 to 16777216 bytes limit for incoming gRPC messages has changed from 4194304 to 104857600 bytes [FEATURE] Distributor/Ingester: Provide ability to not overflow writes in the presence of a leaving or unhealthy ingester. This allows for more efficient ingester rolling restarts. #3305 [FEATURE] Query-frontend: introduced query statistics logged in the query-frontend when enabled via -frontend.query-stats-enabled=true. When enabled, the metric cortex_query_seconds_total is tracked, counting the sum of the wall time spent across all queriers while running queries (on a per-tenant basis). The metrics cortex_request_duration_seconds and cortex_query_seconds_total are different: the first one tracks the request duration (eg. HTTP request from the client), while the latter tracks the sum of the wall time on all queriers involved executing the query. #3539 [ENHANCEMENT] API: Add GZIP HTTP compression to the API responses. Compression can be enabled via -api.response-compression-enabled. #3536 [ENHANCEMENT] Added zone-awareness support on queries. When zone-awareness is enabled, queries will still succeed if all ingesters in a single zone will fail. #3414 [ENHANCEMENT] Blocks storage ingester: exported more TSDB-related metrics. #3412 cortex_ingester_tsdb_wal_corruptions_total cortex_ingester_tsdb_head_truncations_failed_total cortex_ingester_tsdb_head_truncations_total cortex_ingester_tsdb_head_gc_duration_seconds [ENHANCEMENT] Enforced keepalive on all gRPC clients used for inter-service communication. #3431 [ENHANCEMENT] Added cortex_alertmanager_config_hash metric to expose hash of Alertmanager Config loaded per user. #3388 [ENHANCEMENT] Query-Frontend / Query-Scheduler: New component called “Query-Scheduler” has been introduced. Query-Scheduler is simply a queue of requests, moved outside of Query-Frontend. This allows Query-Frontend to be scaled separately from number of queues. To make Query-Frontend and Querier use Query-Scheduler, they need to be started with -frontend.scheduler-address and -querier.scheduler-address options respectively. #3374 #3471 [ENHANCEMENT] Query-frontend / Querier / Ruler: added -querier.max-query-lookback to limit how long back data (series and metadata) can be queried. This setting can be overridden on a per-tenant basis and is enforced in the query-frontend, querier and ruler. #3452 #3458 [ENHANCEMENT] Querier: added -querier.query-store-for-labels-enabled to query store for label names, label values and series APIs. Only works with blocks storage engine. #3461 #3520 [ENHANCEMENT] Ingester: exposed -blocks-storage.tsdb.wal-segment-size-bytes config option to customise the TSDB WAL segment max size. #3476 [ENHANCEMENT] Compactor: concurrently run blocks cleaner for multiple tenants. Concurrency can be configured via -compactor.cleanup-concurrency. #3483 [ENHANCEMENT] Compactor: shuffle tenants before running compaction. #3483 [ENHANCEMENT] Compactor: wait for a stable ring at startup, when sharding is enabled. #3484 [ENHANCEMENT] Store-gateway: added -blocks-storage.bucket-store.index-header-lazy-loading-enabled to enable index-header lazy loading (experimental). When enabled, index-headers will be mmap-ed only once required by a query and will be automatically released after -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout time of inactivity. #3498 [ENHANCEMENT] Alertmanager: added metrics cortex_alertmanager_notification_requests_total and cortex_alertmanager_notification_requests_failed_total. #3518 [ENHANCEMENT] Ingester: added -blocks-storage.tsdb.head-chunks-write-buffer-size-bytes to fine-tune the TSDB head chunks write buffer size when running Cortex blocks storage. #3518 [ENHANCEMENT] /metrics now supports OpenMetrics output. HTTP and gRPC servers metrics can now include exemplars. #3524 [ENHANCEMENT] Expose gRPC keepalive policy options by gRPC server. #3524 [ENHANCEMENT] Blocks storage: enabled caching of meta.json attributes, configurable via -blocks-storage.bucket-store.metadata-cache.metafile-attributes-ttl. #3528 [ENHANCEMENT] Compactor: added a config validation check to fail fast if the compactor has been configured invalid block range periods (each period is expected to be a multiple of the previous one). #3534 [ENHANCEMENT] Blocks storage: concurrently fetch deletion marks from object storage. #3538 [ENHANCEMENT] Blocks storage ingester: ingester can now close idle TSDB and delete local data. #3491 #3552 [ENHANCEMENT] Blocks storage: add option to use V2 signatures for S3 authentication. #3540 [ENHANCEMENT] Exported process metrics to monitor the number of memory map areas allocated. #3537 process_memory_map_areas process_memory_map_areas_limit [ENHANCEMENT] Ruler: Expose gRPC client options. #3523 [ENHANCEMENT] Compactor: added metrics to track on-going compaction. #3535 cortex_compactor_tenants_discovered cortex_compactor_tenants_skipped cortex_compactor_tenants_processing_succeeded cortex_compactor_tenants_processing_failed [ENHANCEMENT] Added new experimental API endpoints: POST /purger/delete_tenant and GET /purger/delete_tenant_status for deleting all tenant data. Only works with blocks storage. Compactor removes blocks that belong to user marked for deletion. #3549 #3558 [ENHANCEMENT] Chunks storage: add option to use V2 signatures for S3 authentication. #3560 [ENHANCEMENT] HA Tracker: Added new limit ha_max_clusters to set the max number of clusters tracked for single user. This limit is disabled by default. #3668 [BUGFIX] Query-Frontend: cortex_query_seconds_total now return seconds not nanoseconds. #3589 [BUGFIX] Blocks storage ingester: fixed some cases leading to a TSDB WAL corruption after a partial write to disk. #3423 [BUGFIX] Blocks storage: Fix the race between ingestion and /flush call resulting in overlapping blocks. #3422 [BUGFIX] Querier: fixed -querier.max-query-into-future which wasn’t correctly enforced on range queries. #3452 [BUGFIX] Fixed float64 precision stability when aggregating metrics before exposing them. This could have lead to false counters resets when querying some metrics exposed by Cortex. #3506 [BUGFIX] Querier: the meta.json sync concurrency done when running Cortex with the blocks storage is now controlled by -blocks-storage.bucket-store.meta-sync-concurrency instead of the incorrect -blocks-storage.bucket-store.block-sync-concurrency (default values are the same). #3531 [BUGFIX] Querier: fixed initialization order of querier module when using blocks storage. It now (again) waits until blocks have been synchronized. #3551 Blocksconvert [ENHANCEMENT] Scheduler: ability to ignore users based on regexp, using -scheduler.ignore-users-regex flag. #3477 [ENHANCEMENT] Builder: Parallelize reading chunks in the final stage of building block. #3470 [ENHANCEMENT] Builder: remove duplicate label names from chunk. #3547 1.5.0 / 2020-11-09 Cortex [CHANGE] Blocks storage: update the default HTTP configuration values for the S3 client to the upstream Thanos default values. #3244 -blocks-storage.s3.http.idle-conn-timeout is set 90 seconds. -blocks-storage.s3.http.response-header-timeout is set to 2 minutes. [CHANGE] Improved shuffle sharding support in the write path. This work introduced some config changes: #3090 Introduced -distributor.sharding-strategy CLI flag (and its respective sharding_strategy YAML config option) to explicitly specify which sharding strategy should be used in the write path -experimental.distributor.user-subring-size flag renamed to -distributor.ingestion-tenant-shard-size user_subring_size limit YAML config option renamed to ingestion_tenant_shard_size [CHANGE] Dropped “blank Alertmanager configuration; using fallback” message from Info to Debug level. #3205 [CHANGE] Zone-awareness replication for time-series now should be explicitly enabled in the distributor via the -distributor.zone-awareness-enabled CLI flag (or its respective YAML config option). Before, zone-aware replication was implicitly enabled if a zone was set on ingesters. #3200 [CHANGE] Removed the deprecated CLI flag -config-yaml. You should use -schema-config-file instead. #3225 [CHANGE] Enforced the HTTP method required by some API endpoints which did (incorrectly) allow any method before that. #3228 GET / GET /config GET /debug/fgprof GET /distributor/all_user_stats GET /distributor/ha_tracker GET /all_user_stats GET /ha-tracker GET /api/v1/user_stats GET /api/v1/chunks GET \u003clegacy-http-prefix\u003e/user_stats GET \u003clegacy-http-prefix\u003e/chunks GET /services GET /multitenant_alertmanager/status GET /status (alertmanager microservice) GET|POST /ingester/ring GET|POST /ring GET|POST /store-gateway/ring GET|POST /compactor/ring GET|POST /ingester/flush GET|POST /ingester/shutdown GET|POST /flush GET|POST /shutdown GET|POST /ruler/ring POST /api/v1/push POST \u003clegacy-http-prefix\u003e/push POST /push POST /ingester/push [CHANGE] Renamed CLI flags to configure the network interface names from which automatically detect the instance IP. #3295 -compactor.ring.instance-interface renamed to -compactor.ring.instance-interface-names -store-gateway.sharding-ring.instance-interface renamed to -store-gateway.sharding-ring.instance-interface-names -distributor.ring.instance-interface renamed to -distributor.ring.instance-interface-names -ruler.ring.instance-interface renamed to -ruler.ring.instance-interface-names [CHANGE] Renamed -\u003cprefix\u003e.redis.enable-tls CLI flag to -\u003cprefix\u003e.redis.tls-enabled, and its respective YAML config option from enable_tls to tls_enabled. #3298 [CHANGE] Increased default -\u003cprefix\u003e.redis.timeout from 100ms to 500ms. #3301 [CHANGE] cortex_alertmanager_config_invalid has been removed in favor of cortex_alertmanager_config_last_reload_successful. #3289 [CHANGE] Query-frontend: POST requests whose body size exceeds 10MiB will be rejected. The max body size can be customised via -frontend.max-body-size. #3276 [FEATURE] Shuffle sharding: added support for shuffle-sharding queriers in the query-frontend. When configured (-frontend.max-queriers-per-tenant globally, or using per-tenant limit max_queriers_per_tenant), each tenants’s requests will be handled by different set of queriers. #3113 #3257 [FEATURE] Shuffle sharding: added support for shuffle-sharding ingesters on the read path. When ingesters shuffle-sharding is enabled and -querier.shuffle-sharding-ingesters-lookback-period is set, queriers will fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since ’now - lookback period’. #3252 [FEATURE] Query-frontend: added compression config to support results cache with compression. #3217 [FEATURE] Add OpenStack Swift support to blocks storage. #3303 [FEATURE] Added support for applying Prometheus relabel configs on series received by the distributor. A metric_relabel_configs field has been added to the per-tenant limits configuration. #3329 [FEATURE] Support for Cassandra client SSL certificates. #3384 [ENHANCEMENT] Ruler: Introduces two new limits -ruler.max-rules-per-rule-group and -ruler.max-rule-groups-per-tenant to control the number of rules per rule group and the total number of rule groups for a given user. They are disabled by default. #3366 [ENHANCEMENT] Allow to specify multiple comma-separated Cortex services to -target CLI option (or its respective YAML config option). For example, -target=all,compactor can be used to start Cortex single-binary with compactor as well. #3275 [ENHANCEMENT] Expose additional HTTP configs for the S3 backend client. New flag are listed below: #3244 -blocks-storage.s3.http.idle-conn-timeout -blocks-storage.s3.http.response-header-timeout -blocks-storage.s3.http.insecure-skip-verify [ENHANCEMENT] Added cortex_query_frontend_connected_clients metric to show the number of workers currently connected to the frontend. #3207 [ENHANCEMENT] Shuffle sharding: improved shuffle sharding in the write path. Shuffle sharding now should be explicitly enabled via -distributor.sharding-strategy CLI flag (or its respective YAML config option) and guarantees stability, consistency, shuffling and balanced zone-awareness properties. #3090 #3214 [ENHANCEMENT] Ingester: added new metric cortex_ingester_active_series to track active series more accurately. Also added options to control whether active series tracking is enabled (-ingester.active-series-metrics-enabled, defaults to false), and how often this metric is updated (-ingester.active-series-metrics-update-period) and max idle time for series to be considered inactive (-ingester.active-series-metrics-idle-timeout). #3153 [ENHANCEMENT] Store-gateway: added zone-aware replication support to blocks replication in the store-gateway. #3200 [ENHANCEMENT] Store-gateway: exported new metrics. #3231 cortex_bucket_store_cached_series_fetch_duration_seconds cortex_bucket_store_cached_postings_fetch_duration_seconds cortex_bucket_stores_gate_queries_max [ENHANCEMENT] Added -version flag to Cortex. #3233 [ENHANCEMENT] Hash ring: added instance registered timestamp to the ring. #3248 [ENHANCEMENT] Reduce tail latency by smoothing out spikes in rate of chunk flush operations. #3191 [ENHANCEMENT] User Cortex as User Agent in http requests issued by Configs DB client. #3264 [ENHANCEMENT] Experimental Ruler API: Fetch rule groups from object storage in parallel. #3218 [ENHANCEMENT] Chunks GCS object storage client uses the fields selector to limit the payload size when listing objects in the bucket. #3218 #3292 [ENHANCEMENT] Added shuffle sharding support to ruler. Added new metric cortex_ruler_sync_rules_total. #3235 [ENHANCEMENT] Return an explicit error when the store-gateway is explicitly requested without a blocks storage engine. #3287 [ENHANCEMENT] Ruler: only load rules that belong to the ruler. Improves rules syncing performances when ruler sharding is enabled. #3269 [ENHANCEMENT] Added -\u003cprefix\u003e.redis.tls-insecure-skip-verify flag. #3298 [ENHANCEMENT] Added cortex_alertmanager_config_last_reload_successful_seconds metric to show timestamp of last successful AM config reload. #3289 [ENHANCEMENT] Blocks storage: reduced number of bucket listing operations to list block content (applies to newly created blocks only). #3363 [ENHANCEMENT] Ruler: Include the tenant ID on the notifier logs. #3372 [ENHANCEMENT] Blocks storage Compactor: Added -compactor.enabled-tenants and -compactor.disabled-tenants to explicitly enable or disable compaction of specific tenants. #3385 [ENHANCEMENT] Blocks storage ingester: Creating checkpoint only once even when there are multiple Head compactions in a single Compact() call. #3373 [BUGFIX] Blocks storage ingester: Read repair memory-mapped chunks file which can end up being empty on abrupt shutdowns combined with faulty disks. #3373 [BUGFIX] Blocks storage ingester: Close TSDB resources on failed startup preventing ingester OOMing. #3373 [BUGFIX] No-longer-needed ingester operations for queries triggered by queriers and rulers are now canceled. #3178 [BUGFIX] Ruler: directories in the configured rules-path will be removed on startup and shutdown in order to ensure they don’t persist between runs. #3195 [BUGFIX] Handle hash-collisions in the query path. #3192 [BUGFIX] Check for postgres rows errors. #3197 [BUGFIX] Ruler Experimental API: Don’t allow rule groups without names or empty rule groups. #3210 [BUGFIX] Experimental Alertmanager API: Do not allow empty Alertmanager configurations or bad template filenames to be submitted through the configuration API. #3185 [BUGFIX] Reduce failures to update heartbeat when using Consul. #3259 [BUGFIX] When using ruler sharding, moving all user rule groups from ruler to a different one and then back could end up with some user groups not being evaluated at all. #3235 [BUGFIX] Fixed shuffle sharding consistency when zone-awareness is enabled and the shard size is increased or instances in a new zone are added. #3299 [BUGFIX] Use a valid grpc header when logging IP addresses. #3307 [BUGFIX] Fixed the metric cortex_prometheus_rule_group_duration_seconds in the Ruler, it wouldn’t report any values. #3310 [BUGFIX] Fixed gRPC connections leaking in rulers when rulers sharding is enabled and APIs called. #3314 [BUGFIX] Fixed shuffle sharding consistency when zone-awareness is enabled and the shard size is increased or instances in a new zone are added. #3299 [BUGFIX] Fixed Gossip memberlist members joining when addresses are configured using DNS-based service discovery. #3360 [BUGFIX] Ingester: fail to start an ingester running the blocks storage, if unable to load any existing TSDB at startup. #3354 [BUGFIX] Blocks storage: Avoid deletion of blocks in the ingester which are not shipped to the storage yet. #3346 [BUGFIX] Fix common prefixes returned by List method of S3 client. #3358 [BUGFIX] Honor configured timeout in Azure and GCS object clients. #3285 [BUGFIX] Blocks storage: Avoid creating blocks larger than configured block range period on forced compaction and when TSDB is idle. #3344 [BUGFIX] Shuffle sharding: fixed max global series per user/metric limit when shuffle sharding and -distributor.shard-by-all-labels=true are both enabled in distributor. When using these global limits you should now set -distributor.sharding-strategy and -distributor.zone-awareness-enabled to ingesters too. #3369 [BUGFIX] Slow query logging: when using downstream server request parameters were not logged. #3276 [BUGFIX] Fixed tenant detection in the ruler and alertmanager API when running without auth. #3343 Blocksconvert [ENHANCEMENT] Blocksconvert – Builder: download plan file locally before processing it. #3209 [ENHANCEMENT] Blocksconvert – Cleaner: added new tool for deleting chunks data. #3283 [ENHANCEMENT] Blocksconvert – Scanner: support for scanning specific date-range only. #3222 [ENHANCEMENT] Blocksconvert – Scanner: metrics for tracking progress. #3222 [ENHANCEMENT] Blocksconvert – Builder: retry block upload before giving up. #3245 [ENHANCEMENT] Blocksconvert – Scanner: upload plans concurrently. #3340 [BUGFIX] Blocksconvert: fix chunks ordering in the block. Chunks in different order than series work just fine in TSDB blocks at the moment, but it’s not consistent with what Prometheus does and future Prometheus and Cortex optimizations may rely on this ordering. #3371 1.4.0 / 2020-10-02 [CHANGE] TLS configuration for gRPC, HTTP and etcd clients is now marked as experimental. These features are not yet fully baked, and we expect possible small breaking changes in Cortex 1.5. #3198 [CHANGE] Cassandra backend support is now GA (stable). #3180 [CHANGE] Blocks storage is now GA (stable). The -experimental prefix has been removed from all CLI flags related to the blocks storage (no YAML config changes). #3180 #3201 -experimental.blocks-storage.* flags renamed to -blocks-storage.* -experimental.store-gateway.* flags renamed to -store-gateway.* -experimental.querier.store-gateway-client.* flags renamed to -querier.store-gateway-client.* -experimental.querier.store-gateway-addresses flag renamed to -querier.store-gateway-addresses -store-gateway.replication-factor flag renamed to -store-gateway.sharding-ring.replication-factor -store-gateway.tokens-file-path flag renamed to store-gateway.sharding-ring.tokens-file-path [CHANGE] Ingester: Removed deprecated untyped record from chunks WAL. Only if you are running v1.0 or below, it is recommended to first upgrade to v1.1/v1.2/v1.3 and run it for a day before upgrading to v1.4 to avoid data loss. #3115 [CHANGE] Distributor API endpoints are no longer served unless target is set to distributor or all. #3112 [CHANGE] Increase the default Cassandra client replication factor to 3. #3007 [CHANGE] Blocks storage: removed the support to transfer blocks between ingesters on shutdown. When running the Cortex blocks storage, ingesters are expected to run with a persistent disk. The following metrics have been removed: #2996 cortex_ingester_sent_files cortex_ingester_received_files cortex_ingester_received_bytes_total cortex_ingester_sent_bytes_total [CHANGE] The buckets for the cortex_chunk_store_index_lookups_per_query metric have been changed to 1, 2, 4, 8, 16. #3021 [CHANGE] Blocks storage: the operation label value getrange has changed into get_range for the metrics thanos_store_bucket_cache_operation_requests_total and thanos_store_bucket_cache_operation_hits_total. #3000 [CHANGE] Experimental Delete Series: /api/v1/admin/tsdb/delete_series and /api/v1/admin/tsdb/cancel_delete_request purger APIs to return status code 204 instead of 200 for success. #2946 [CHANGE] Histogram cortex_memcache_request_duration_seconds method label value changes from Memcached.Get to Memcached.GetBatched for batched lookups, and is not reported for non-batched lookups (label value Memcached.GetMulti remains, and had exactly the same value as Get in nonbatched lookups). The same change applies to tracing spans. #3046 [CHANGE] TLS server validation is now enabled by default, a new parameter tls_insecure_skip_verify can be set to true to skip validation optionally. #3030 [CHANGE] cortex_ruler_config_update_failures_total has been removed in favor of cortex_ruler_config_last_reload_successful. #3056 [CHANGE] ruler.evaluation_delay_duration field in YAML config has been moved and renamed to limits.ruler_evaluation_delay_duration. #3098 [CHANGE] Removed obsolete results_cache.max_freshness from YAML config (deprecated since Cortex 1.2). #3145 [CHANGE] Removed obsolete -promql.lookback-delta option (deprecated since Cortex 1.2, replaced with -querier.lookback-delta). #3144 [CHANGE] Cache: added support for Redis Cluster and Redis Sentinel. #2961 The following changes have been made in Redis configuration: -redis.master_name added -redis.db added -redis.max-active-conns changed to -redis.pool-size -redis.max-conn-lifetime changed to -redis.max-connection-age -redis.max-idle-conns removed -redis.wait-on-pool-exhaustion removed [CHANGE] TLS configuration for gRPC, HTTP and etcd clients is now marked as experimental. These features are not yet fully baked, and we expect possible small breaking changes in Cortex 1.5. #3198 [CHANGE] Fixed store-gateway CLI flags inconsistencies. #3201 -store-gateway.replication-factor flag renamed to -store-gateway.sharding-ring.replication-factor -store-gateway.tokens-file-path flag renamed to store-gateway.sharding-ring.tokens-file-path [FEATURE] Logging of the source IP passed along by a reverse proxy is now supported by setting the -server.log-source-ips-enabled. For non standard headers the settings -server.log-source-ips-header and -server.log-source-ips-regex can be used. #2985 [FEATURE] Blocks storage: added shuffle sharding support to store-gateway blocks sharding. Added the following additional metrics to store-gateway: #3069 cortex_bucket_stores_tenants_discovered cortex_bucket_stores_tenants_synced [FEATURE] Experimental blocksconvert: introduce an experimental tool blocksconvert to migrate long-term storage chunks to blocks. #3092 #3122 #3127 #3162 [ENHANCEMENT] Improve the Alertmanager logging when serving requests from its API / UI. #3397 [ENHANCEMENT] Add support for azure storage in China, German and US Government environments. #2988 [ENHANCEMENT] Query-tee: added a small tolerance to floating point sample values comparison. #2994 [ENHANCEMENT] Query-tee: add support for doing a passthrough of requests to preferred backend for unregistered routes #3018 [ENHANCEMENT] Expose storage.aws.dynamodb.backoff_config configuration file field. #3026 [ENHANCEMENT] Added cortex_request_message_bytes and cortex_response_message_bytes histograms to track received and sent gRPC message and HTTP request/response sizes. Added cortex_inflight_requests gauge to track number of inflight gRPC and HTTP requests. #3064 [ENHANCEMENT] Publish ruler’s ring metrics. #3074 [ENHANCEMENT] Add config validation to the experimental Alertmanager API. Invalid configs are no longer accepted. #3053 [ENHANCEMENT] Add “integration” as a label for cortex_alertmanager_notifications_total and cortex_alertmanager_notifications_failed_total metrics. #3056 [ENHANCEMENT] Add cortex_ruler_config_last_reload_successful and cortex_ruler_config_last_reload_successful_seconds to check status of users rule manager. #3056 [ENHANCEMENT] The configuration validation now fails if an empty YAML node has been set for a root YAML config property. #3080 [ENHANCEMENT] Memcached dial() calls now have a circuit-breaker to avoid hammering a broken cache. #3051, #3189 [ENHANCEMENT] -ruler.evaluation-delay-duration is now overridable as a per-tenant limit, ruler_evaluation_delay_duration. #3098 [ENHANCEMENT] Add TLS support to etcd client. #3102 [ENHANCEMENT] When a tenant accesses the Alertmanager UI or its API, if we have valid -alertmanager.configs.fallback we’ll use that to start the manager and avoid failing the request. #3073 [ENHANCEMENT] Add DELETE api/v1/rules/{namespace} to the Ruler. It allows all the rule groups of a namespace to be deleted. #3120 [ENHANCEMENT] Experimental Delete Series: Retry processing of Delete requests during failures. #2926 [ENHANCEMENT] Improve performance of QueryStream() in ingesters. #3177 [ENHANCEMENT] Modules included in “All” target are now visible in output of -modules CLI flag. #3155 [ENHANCEMENT] Added /debug/fgprof endpoint to debug running Cortex process using fgprof. This adds up to the existing /debug/... endpoints. #3131 [ENHANCEMENT] Blocks storage: optimised /api/v1/series for blocks storage. (#2976) [BUGFIX] Ruler: when loading rules from “local” storage, check for directory after resolving symlink. #3137 [BUGFIX] Query-frontend: Fixed rounding for incoming query timestamps, to be 100% Prometheus compatible. #2990 [BUGFIX] Querier: Merge results from chunks and blocks ingesters when using streaming of results. #3013 [BUGFIX] Querier: query /series from ingesters regardless the -querier.query-ingesters-within setting. #3035 [BUGFIX] Blocks storage: Ingester is less likely to hit gRPC message size limit when streaming data to queriers. #3015 [BUGFIX] Blocks storage: fixed memberlist support for the store-gateways and compactors ring used when blocks sharding is enabled. #3058 #3095 [BUGFIX] Fix configuration for TLS server validation, TLS skip verify was hardcoded to true for all TLS configurations and prevented validation of server certificates. #3030 [BUGFIX] Fixes the Alertmanager panicking when no -alertmanager.web.external-url is provided. #3017 [BUGFIX] Fixes the registration of the Alertmanager API metrics cortex_alertmanager_alerts_received_total and cortex_alertmanager_alerts_invalid_total. #3065 [BUGFIX] Fixes flag needs an argument: -config.expand-env error. #3087 [BUGFIX] An index optimisation actually slows things down when using caching. Moved it to the right location. #2973 [BUGFIX] Ingester: If push request contained both valid and invalid samples, valid samples were ingested but not stored to WAL of the chunks storage. This has been fixed. #3067 [BUGFIX] Cassandra: fixed consistency setting in the CQL session when creating the keyspace. #3105 [BUGFIX] Ruler: Config API would return both the record and alert in YAML response keys even when one of them must be empty. #3120 [BUGFIX] Index page now uses configured HTTP path prefix when creating links. #3126 [BUGFIX] Purger: fixed deadlock when reloading of tombstones failed. #3182 [BUGFIX] Fixed panic in flusher job, when error writing chunks to the store would cause “idle” chunks to be flushed, which triggered panic. #3140 [BUGFIX] Index page no longer shows links that are not valid for running Cortex instance. #3133 [BUGFIX] Configs: prevent validation of templates to fail when using template functions. #3157 [BUGFIX] Configuring the S3 URL with an @ but without username and password doesn’t enable the AWS static credentials anymore. #3170 [BUGFIX] Limit errors on ranged queries (api/v1/query_range) no longer return a status code 500 but 422 instead. #3167 [BUGFIX] Handle hash-collisions in the query path. Before this fix, Cortex could occasionally mix up two different series in a query, leading to invalid results, when -querier.ingester-streaming was used. #3192 1.3.0 / 2020-08-21 [CHANGE] Replace the metric cortex_alertmanager_configs with cortex_alertmanager_config_invalid exposed by Alertmanager. #2960 [CHANGE] Experimental Delete Series: Change target flag for purger from data-purger to purger. #2777 [CHANGE] Experimental blocks storage: The max concurrent queries against the long-term storage, configured via -experimental.blocks-storage.bucket-store.max-concurrent, is now a limit shared across all tenants and not a per-tenant limit anymore. The default value has changed from 20 to 100 and the following new metrics have been added: #2797 cortex_bucket_stores_gate_queries_concurrent_max cortex_bucket_stores_gate_queries_in_flight cortex_bucket_stores_gate_duration_seconds [CHANGE] Metric cortex_ingester_flush_reasons has been renamed to cortex_ingester_flushing_enqueued_series_total, and new metric cortex_ingester_flushing_dequeued_series_total with outcome label (superset of reason) has been added. #2802 #2818 #2998 [CHANGE] Experimental Delete Series: Metric cortex_purger_oldest_pending_delete_request_age_seconds would track age of delete requests since they are over their cancellation period instead of their creation time. #2806 [CHANGE] Experimental blocks storage: the store-gateway service is required in a Cortex cluster running with the experimental blocks storage. Removed the -experimental.tsdb.store-gateway-enabled CLI flag and store_gateway_enabled YAML config option. The store-gateway is now always enabled when the storage engine is blocks. #2822 [CHANGE] Experimental blocks storage: removed support for -experimental.blocks-storage.bucket-store.max-sample-count flag because the implementation was flawed. To limit the number of samples/chunks processed by a single query you can set -store.query-chunk-limit, which is now supported by the blocks storage too. #2852 [CHANGE] Ingester: Chunks flushed via /flush stay in memory until retention period is reached. This affects cortex_ingester_memory_chunks metric. #2778 [CHANGE] Querier: the error message returned when the query time range exceeds -store.max-query-length has changed from invalid query, length \u003e limit (X \u003e Y) to the query time range exceeds the limit (query length: X, limit: Y). #2826 [CHANGE] Add component label to metrics exposed by chunk, delete and index store clients. #2774 [CHANGE] Querier: when -querier.query-ingesters-within is configured, the time range of the query sent to ingesters is now manipulated to ensure the query start time is not older than ’now - query-ingesters-within’. #2904 [CHANGE] KV: The role label which was a label of multi KV store client only has been added to metrics of every KV store client. If KV store client is not multi, then the value of role label is primary. #2837 [CHANGE] Added the engine label to the metrics exposed by the Prometheus query engine, to distinguish between ruler and querier metrics. #2854 [CHANGE] Added ruler to the single binary when started with -target=all (default). #2854 [CHANGE] Experimental blocks storage: compact head when opening TSDB. This should only affect ingester startup after it was unable to compact head in previous run. #2870 [CHANGE] Metric cortex_overrides_last_reload_successful has been renamed to cortex_runtime_config_last_reload_successful. #2874 [CHANGE] HipChat support has been removed from the alertmanager (because removed from the Prometheus upstream too). #2902 [CHANGE] Add constant label name to metric cortex_cache_request_duration_seconds. #2903 [CHANGE] Add user label to metric cortex_query_frontend_queue_length. #2939 [CHANGE] Experimental blocks storage: cleaned up the config and renamed “TSDB” to “blocks storage”. #2937 The storage engine setting value has been changed from tsdb to blocks; this affects -store.engine CLI flag and its respective YAML option. The root level YAML config has changed from tsdb to blocks_storage The prefix of all CLI flags has changed from -experimental.tsdb. to -experimental.blocks-storage. The following settings have been grouped under tsdb property in the YAML config and their CLI flags changed: -experimental.tsdb.dir changed to -experimental.blocks-storage.tsdb.dir -experimental.tsdb.block-ranges-period changed to -experimental.blocks-storage.tsdb.block-ranges-period -experimental.tsdb.retention-period changed to -experimental.blocks-storage.tsdb.retention-period -experimental.tsdb.ship-interval changed to -experimental.blocks-storage.tsdb.ship-interval -experimental.tsdb.ship-concurrency changed to -experimental.blocks-storage.tsdb.ship-concurrency -experimental.tsdb.max-tsdb-opening-concurrency-on-startup changed to -experimental.blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup -experimental.tsdb.head-compaction-interval changed to -experimental.blocks-storage.tsdb.head-compaction-interval -experimental.tsdb.head-compaction-concurrency changed to -experimental.blocks-storage.tsdb.head-compaction-concurrency -experimental.tsdb.head-compaction-idle-timeout changed to -experimental.blocks-storage.tsdb.head-compaction-idle-timeout -experimental.tsdb.stripe-size changed to -experimental.blocks-storage.tsdb.stripe-size -experimental.tsdb.wal-compression-enabled changed to -experimental.blocks-storage.tsdb.wal-compression-enabled -experimental.tsdb.flush-blocks-on-shutdown changed to -experimental.blocks-storage.tsdb.flush-blocks-on-shutdown [CHANGE] Flags -bigtable.grpc-use-gzip-compression, -ingester.client.grpc-use-gzip-compression, -querier.frontend-client.grpc-use-gzip-compression are now deprecated. #2940 [CHANGE] Limit errors reported by ingester during query-time now return HTTP status code 422. #2941 [FEATURE] Introduced ruler.for-outage-tolerance, Max time to tolerate outage for restoring “for” state of alert. #2783 [FEATURE] Introduced ruler.for-grace-period, Minimum duration between alert and restored “for” state. This is maintained only for alerts with configured “for” time greater than grace period. #2783 [FEATURE] Introduced ruler.resend-delay, Minimum amount of time to wait before resending an alert to Alertmanager. #2783 [FEATURE] Ruler: added local filesystem support to store rules (read-only). #2854 [ENHANCEMENT] Upgraded Docker base images to alpine:3.12. #2862 [ENHANCEMENT] Experimental: Querier can now optionally query secondary store. This is specified by using -querier.second-store-engine option, with values chunks or blocks. Standard configuration options for this store are used. Additionally, this querying can be configured to happen only for queries that need data older than -querier.use-second-store-before-time. Default value of zero will always query secondary store. #2747 [ENHANCEMENT] Query-tee: increased the cortex_querytee_request_duration_seconds metric buckets granularity. #2799 [ENHANCEMENT] Query-tee: fail to start if the configured -backend.preferred is unknown. #2799 [ENHANCEMENT] Ruler: Added the following metrics: #2786 cortex_prometheus_notifications_latency_seconds cortex_prometheus_notifications_errors_total cortex_prometheus_notifications_sent_total cortex_prometheus_notifications_dropped_total cortex_prometheus_notifications_queue_length cortex_prometheus_notifications_queue_capacity cortex_prometheus_notifications_alertmanagers_discovered [ENHANCEMENT] The behavior of the /ready was changed for the query frontend to indicate when it was ready to accept queries. This is intended for use by a read path load balancer that would want to wait for the frontend to have attached queriers before including it in the backend. #2733 [ENHANCEMENT] Experimental Delete Series: Add support for deletion of chunks for remaining stores. #2801 [ENHANCEMENT] Add -modules command line flag to list possible values for -target. Also, log warning if given target is internal component. #2752 [ENHANCEMENT] Added -ingester.flush-on-shutdown-with-wal-enabled option to enable chunks flushing even when WAL is enabled. #2780 [ENHANCEMENT] Query-tee: Support for custom API prefix by using -server.path-prefix option. #2814 [ENHANCEMENT] Query-tee: Forward X-Scope-OrgId header to backend, if present in the request. #2815 [ENHANCEMENT] Experimental blocks storage: Added -experimental.blocks-storage.tsdb.head-compaction-idle-timeout option to force compaction of data in memory into a block. #2803 [ENHANCEMENT] Experimental blocks storage: Added support for flushing blocks via /flush, /shutdown (previously these only worked for chunks storage) and by using -experimental.blocks-storage.tsdb.flush-blocks-on-shutdown option. #2794 [ENHANCEMENT] Experimental blocks storage: Added support to enforce max query time range length via -store.max-query-length. #2826 [ENHANCEMENT] Experimental blocks storage: Added support to limit the max number of chunks that can be fetched from the long-term storage while executing a query. The limit is enforced both in the querier and store-gateway, and is configurable via -store.query-chunk-limit. #2852 #2922 [ENHANCEMENT] Ingester: Added new metric cortex_ingester_flush_series_in_progress that reports number of ongoing flush-series operations. Useful when calling /flush handler: if cortex_ingester_flush_queue_length + cortex_ingester_flush_series_in_progress is 0, all flushes are finished. #2778 [ENHANCEMENT] Memberlist members can join cluster via SRV records. #2788 [ENHANCEMENT] Added configuration options for chunks s3 client. #2831 s3.endpoint s3.region s3.access-key-id s3.secret-access-key s3.insecure s3.sse-encryption s3.http.idle-conn-timeout s3.http.response-header-timeout s3.http.insecure-skip-verify [ENHANCEMENT] Prometheus upgraded. #2798 #2849 #2867 #2902 #2918 Optimized labels regex matchers for patterns containing literals (eg. foo.*, .*foo, .*foo.*) [ENHANCEMENT] Add metric cortex_ruler_config_update_failures_total to Ruler to track failures of loading rules files. #2857 [ENHANCEMENT] Experimental Alertmanager: Alertmanager configuration persisted to object storage using an experimental API that accepts and returns YAML-based Alertmanager configuration. #2768 [ENHANCEMENT] Ruler: -ruler.alertmanager-url now supports multiple URLs. Each URL is treated as a separate Alertmanager group. Support for multiple Alertmanagers in a group can be achieved by using DNS service discovery. #2851 [ENHANCEMENT] Experimental blocks storage: Cortex Flusher now works with blocks engine. Flusher needs to be provided with blocks-engine configuration, existing Flusher flags are not used (they are only relevant for chunks engine). Note that flush errors are only reported via log. #2877 [ENHANCEMENT] Flusher: Added -flusher.exit-after-flush option (defaults to true) to control whether Cortex should stop completely after Flusher has finished its work. #2877 [ENHANCEMENT] Added metrics cortex_config_hash and cortex_runtime_config_hash to expose hash of the currently active config file. #2874 [ENHANCEMENT] Logger: added JSON logging support, configured via the -log.format=json CLI flag or its respective YAML config option. #2386 [ENHANCEMENT] Added new flags -bigtable.grpc-compression, -ingester.client.grpc-compression, -querier.frontend-client.grpc-compression to configure compression used by gRPC. Valid values are gzip, snappy, or empty string (no compression, default). #2940 [ENHANCEMENT] Clarify limitations of the /api/v1/series, /api/v1/labels and /api/v1/label/{name}/values endpoints. #2953 [ENHANCEMENT] Ingester: added Dropped outcome to metric cortex_ingester_flushing_dequeued_series_total. #2998 [BUGFIX] Fixed a bug with api/v1/query_range where no responses would return null values for result and empty values for resultType. #2962 [BUGFIX] Fixed a bug in the index intersect code causing storage to return more chunks/series than required. #2796 [BUGFIX] Fixed the number of reported keys in the background cache queue. #2764 [BUGFIX] Fix race in processing of headers in sharded queries. #2762 [BUGFIX] Query Frontend: Do not re-split sharded requests around ingester boundaries. #2766 [BUGFIX] Experimental Delete Series: Fixed a problem with cache generation numbers prefixed to cache keys. #2800 [BUGFIX] Ingester: Flushing chunks via /flush endpoint could previously lead to panic, if chunks were already flushed before and then removed from memory during the flush caused by /flush handler. Immediate flush now doesn’t cause chunks to be flushed again. Samples received during flush triggered via /flush handler are no longer discarded. #2778 [BUGFIX] Prometheus upgraded. #2849 Fixed unknown symbol error during head compaction [BUGFIX] Fix panic when using cassandra as store for both index and delete requests. #2774 [BUGFIX] Experimental Delete Series: Fixed a data race in Purger. #2817 [BUGFIX] KV: Fixed a bug that triggered a panic due to metrics being registered with the same name but different labels when using a multi configured KV client. #2837 [BUGFIX] Query-frontend: Fix passing HTTP Host header if -frontend.downstream-url is configured. #2880 [BUGFIX] Ingester: Improve time-series distribution when -experimental.distributor.user-subring-size is enabled. #2887 [BUGFIX] Set content type to application/x-protobuf for remote_read responses. #2915 [BUGFIX] Fixed ruler and store-gateway instance registration in the ring (when sharding is enabled) when a new instance replaces abruptly terminated one, and the only difference between the two instances is the address. #2954 [BUGFIX] Fixed Missing chunks and index config causing silent failure Absence of chunks and index from schema config is not validated. #2732 [BUGFIX] Fix panic caused by KVs from boltdb being used beyond their life. #2971 [BUGFIX] Experimental blocks storage: /api/v1/series, /api/v1/labels and /api/v1/label/{name}/values only query the TSDB head regardless of the configured -experimental.blocks-storage.tsdb.retention-period. #2974 [BUGFIX] Ingester: Avoid indefinite checkpointing in case of surge in number of series. #2955 [BUGFIX] Querier: query /series from ingesters regardless the -querier.query-ingesters-within setting. #3035 [BUGFIX] Ruler: fixed an unintentional breaking change introduced in the ruler’s alertmanager_url YAML config option, which changed the value from a string to a list of strings. #2989 1.2.0 / 2020-07-01 [CHANGE] Metric cortex_kv_request_duration_seconds now includes name label to denote which client is being used as well as the backend label to denote the KV backend implementation in use. #2648 [CHANGE] Experimental Ruler: Rule groups persisted to object storage using the experimental API have an updated object key encoding to better handle special characters. Rule groups previously-stored using object storage must be renamed to the new format. #2646 [CHANGE] Query Frontend now uses Round Robin to choose a tenant queue to service next. #2553 [CHANGE] -promql.lookback-delta is now deprecated and has been replaced by -querier.lookback-delta along with lookback_delta entry under querier in the config file. -promql.lookback-delta will be removed in v1.4.0. #2604 [CHANGE] Experimental TSDB: removed -experimental.tsdb.bucket-store.binary-index-header-enabled flag. Now the binary index-header is always enabled. [CHANGE] Experimental TSDB: Renamed index-cache metrics to use original metric names from Thanos, as Cortex is not aggregating them in any way: #2627 cortex_\u003cservice\u003e_blocks_index_cache_items_evicted_total =\u003e thanos_store_index_cache_items_evicted_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_items_added_total =\u003e thanos_store_index_cache_items_added_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_requests_total =\u003e thanos_store_index_cache_requests_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_items_overflowed_total =\u003e thanos_store_index_cache_items_overflowed_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_hits_total =\u003e thanos_store_index_cache_hits_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_items =\u003e thanos_store_index_cache_items{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_items_size_bytes =\u003e thanos_store_index_cache_items_size_bytes{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_total_size_bytes =\u003e thanos_store_index_cache_total_size_bytes{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_memcached_operations_total =\u003e thanos_memcached_operations_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_memcached_operation_failures_total =\u003e thanos_memcached_operation_failures_total{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_memcached_operation_duration_seconds =\u003e thanos_memcached_operation_duration_seconds{name=\"index-cache\"} cortex_\u003cservice\u003e_blocks_index_cache_memcached_operation_skipped_total =\u003e thanos_memcached_operation_skipped_total{name=\"index-cache\"} [CHANGE] Experimental TSDB: Renamed metrics in bucket stores: #2627 cortex_\u003cservice\u003e_blocks_meta_syncs_total =\u003e cortex_blocks_meta_syncs_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_blocks_meta_sync_failures_total =\u003e cortex_blocks_meta_sync_failures_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_blocks_meta_sync_duration_seconds =\u003e cortex_blocks_meta_sync_duration_seconds{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_blocks_meta_sync_consistency_delay_seconds =\u003e cortex_blocks_meta_sync_consistency_delay_seconds{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_blocks_meta_synced =\u003e cortex_blocks_meta_synced{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_block_loads_total =\u003e cortex_bucket_store_block_loads_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_block_load_failures_total =\u003e cortex_bucket_store_block_load_failures_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_block_drops_total =\u003e cortex_bucket_store_block_drops_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_block_drop_failures_total =\u003e cortex_bucket_store_block_drop_failures_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_blocks_loaded =\u003e cortex_bucket_store_blocks_loaded{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_data_touched =\u003e cortex_bucket_store_series_data_touched{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_data_fetched =\u003e cortex_bucket_store_series_data_fetched{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_data_size_touched_bytes =\u003e cortex_bucket_store_series_data_size_touched_bytes{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_data_size_fetched_bytes =\u003e cortex_bucket_store_series_data_size_fetched_bytes{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_blocks_queried =\u003e cortex_bucket_store_series_blocks_queried{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_get_all_duration_seconds =\u003e cortex_bucket_store_series_get_all_duration_seconds{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_merge_duration_seconds =\u003e cortex_bucket_store_series_merge_duration_seconds{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_refetches_total =\u003e cortex_bucket_store_series_refetches_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_series_result_series =\u003e cortex_bucket_store_series_result_series{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_cached_postings_compressions_total =\u003e cortex_bucket_store_cached_postings_compressions_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_cached_postings_compression_errors_total =\u003e cortex_bucket_store_cached_postings_compression_errors_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_cached_postings_compression_time_seconds =\u003e cortex_bucket_store_cached_postings_compression_time_seconds{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_cached_postings_original_size_bytes_total =\u003e cortex_bucket_store_cached_postings_original_size_bytes_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_bucket_store_cached_postings_compressed_size_bytes_total =\u003e cortex_bucket_store_cached_postings_compressed_size_bytes_total{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_blocks_sync_seconds =\u003e cortex_bucket_stores_blocks_sync_seconds{component=\"\u003cservice\u003e\"} cortex_\u003cservice\u003e_blocks_last_successful_sync_timestamp_seconds =\u003e cortex_bucket_stores_blocks_last_successful_sync_timestamp_seconds{component=\"\u003cservice\u003e\"} [CHANGE] Available command-line flags are printed to stdout, and only when requested via -help. Using invalid flag no longer causes printing of all available flags. #2691 [CHANGE] Experimental Memberlist ring: randomize gossip node names to avoid conflicts when running multiple clients on the same host, or reusing host names (eg. pods in statefulset). Node name randomization can be disabled by using -memberlist.randomize-node-name=false. #2715 [CHANGE] Memberlist KV client is no longer considered experimental. #2725 [CHANGE] Experimental Delete Series: Make delete request cancellation duration configurable. #2760 [CHANGE] Removed -store.fullsize-chunks option which was undocumented and unused (it broke ingester hand-overs). #2656 [CHANGE] Query with no metric name that has previously resulted in HTTP status code 500 now returns status code 422 instead. #2571 [FEATURE] TLS config options added for GRPC clients in Querier (Query-frontend client \u0026 Ingester client), Ruler, Store Gateway, as well as HTTP client in Config store client. #2502 [FEATURE] The flag -frontend.max-cache-freshness is now supported within the limits overrides, to specify per-tenant max cache freshness values. The corresponding YAML config parameter has been changed from results_cache.max_freshness to limits_config.max_cache_freshness. The legacy YAML config parameter (results_cache.max_freshness) will continue to be supported till Cortex release v1.4.0. #2609 [FEATURE] Experimental gRPC Store: Added support to 3rd parties index and chunk stores using gRPC client/server plugin mechanism. #2220 [FEATURE] Add -cassandra.table-options flag to customize table options of Cassandra when creating the index or chunk table. #2575 [ENHANCEMENT] Propagate GOPROXY value when building build-image. This is to help the builders building the code in a Network where default Go proxy is not accessible (e.g. when behind some corporate VPN). #2741 [ENHANCEMENT] Querier: Added metric cortex_querier_request_duration_seconds for all requests to the querier. #2708 [ENHANCEMENT] Cortex is now built with Go 1.14. #2480 #2749 #2753 [ENHANCEMENT] Experimental TSDB: added the following metrics to the ingester: #2580 #2583 #2589 #2654 cortex_ingester_tsdb_appender_add_duration_seconds cortex_ingester_tsdb_appender_commit_duration_seconds cortex_ingester_tsdb_refcache_purge_duration_seconds cortex_ingester_tsdb_compactions_total cortex_ingester_tsdb_compaction_duration_seconds cortex_ingester_tsdb_wal_fsync_duration_seconds cortex_ingester_tsdb_wal_page_flushes_total cortex_ingester_tsdb_wal_completed_pages_total cortex_ingester_tsdb_wal_truncations_failed_total cortex_ingester_tsdb_wal_truncations_total cortex_ingester_tsdb_wal_writes_failed_total cortex_ingester_tsdb_checkpoint_deletions_failed_total cortex_ingester_tsdb_checkpoint_deletions_total cortex_ingester_tsdb_checkpoint_creations_failed_total cortex_ingester_tsdb_checkpoint_creations_total cortex_ingester_tsdb_wal_truncate_duration_seconds cortex_ingester_tsdb_head_active_appenders cortex_ingester_tsdb_head_series_not_found_total cortex_ingester_tsdb_head_chunks cortex_ingester_tsdb_mmap_chunk_corruptions_total cortex_ingester_tsdb_head_chunks_created_total cortex_ingester_tsdb_head_chunks_removed_total [ENHANCEMENT] Experimental TSDB: added metrics useful to alert on critical conditions of the blocks storage: #2573 cortex_compactor_last_successful_run_timestamp_seconds cortex_querier_blocks_last_successful_sync_timestamp_seconds (when store-gateway is disabled) cortex_querier_blocks_last_successful_scan_timestamp_seconds (when store-gateway is enabled) cortex_storegateway_blocks_last_successful_sync_timestamp_seconds [ENHANCEMENT] Experimental TSDB: added the flag -experimental.tsdb.wal-compression-enabled to allow to enable TSDB WAL compression. #2585 [ENHANCEMENT] Experimental TSDB: Querier and store-gateway components can now use so-called “caching bucket”, which can currently cache fetched chunks into shared memcached server. #2572 [ENHANCEMENT] Ruler: Automatically remove unhealthy rulers from the ring. #2587 [ENHANCEMENT] Query-tee: added support to /metadata, /alerts, and /rules endpoints #2600 [ENHANCEMENT] Query-tee: added support to query results comparison between two different backends. The comparison is disabled by default and can be enabled via -proxy.compare-responses=true. #2611 [ENHANCEMENT] Query-tee: improved the query-tee to not wait all backend responses before sending back the response to the client. The query-tee now sends back to the client first successful response, while honoring the -backend.preferred option. #2702 [ENHANCEMENT] Thanos and Prometheus upgraded. #2602 #2604 #2634 #2659 #2686 #2756 TSDB now holds less WAL files after Head Truncation. TSDB now does memory-mapping of Head chunks and reduces memory usage. [ENHANCEMENT] Experimental TSDB: decoupled blocks deletion from blocks compaction in the compactor, so that blocks deletion is not blocked by a busy compactor. The following metrics have been added: #2623 cortex_compactor_block_cleanup_started_total cortex_compactor_block_cleanup_completed_total cortex_compactor_block_cleanup_failed_total cortex_compactor_block_cleanup_last_successful_run_timestamp_seconds [ENHANCEMENT] Experimental TSDB: Use shared cache for metadata. This is especially useful when running multiple querier and store-gateway components to reduce number of object store API calls. #2626 #2640 [ENHANCEMENT] Experimental TSDB: when -querier.query-store-after is configured and running the experimental blocks storage, the time range of the query sent to the store is now manipulated to ensure the query end time is not more recent than ’now - query-store-after’. #2642 [ENHANCEMENT] Experimental TSDB: small performance improvement in concurrent usage of RefCache, used during samples ingestion. #2651 [ENHANCEMENT] The following endpoints now respond appropriately to an Accept header with the value application/json #2673 /distributor/all_user_stats /distributor/ha_tracker /ingester/ring /store-gateway/ring /compactor/ring /ruler/ring /services [ENHANCEMENT] Experimental Cassandra backend: Add -cassandra.num-connections to allow increasing the number of TCP connections to each Cassandra server. #2666 [ENHANCEMENT] Experimental Cassandra backend: Use separate Cassandra clients and connections for reads and writes. #2666 [ENHANCEMENT] Experimental Cassandra backend: Add -cassandra.reconnect-interval to allow specifying the reconnect interval to a Cassandra server that has been marked DOWN by the gocql driver. Also change the default value of the reconnect interval from 60s to 1s. #2687 [ENHANCEMENT] Experimental Cassandra backend: Add option -cassandra.convict-hosts-on-failure=false to not convict host of being down when a request fails. #2684 [ENHANCEMENT] Experimental TSDB: Applied a jitter to the period bucket scans in order to better distribute bucket operations over the time and increase the probability of hitting the shared cache (if configured). #2693 [ENHANCEMENT] Experimental TSDB: Series limit per user and per metric now work in TSDB blocks. #2676 [ENHANCEMENT] Experimental Memberlist: Added ability to periodically rejoin the memberlist cluster. #2724 [ENHANCEMENT] Experimental Delete Series: Added the following metrics for monitoring processing of delete requests: #2730 cortex_purger_load_pending_requests_attempts_total: Number of attempts that were made to load pending requests with status. cortex_purger_oldest_pending_delete_request_age_seconds: Age of oldest pending delete request in seconds. cortex_purger_pending_delete_requests_count: Count of requests which are in process or are ready to be processed. [ENHANCEMENT] Experimental TSDB: Improved compactor to hard-delete also partial blocks with an deletion mark (even if the deletion mark threshold has not been reached). #2751 [ENHANCEMENT] Experimental TSDB: Introduced a consistency check done by the querier to ensure all expected blocks have been queried via the store-gateway. If a block is missing on a store-gateway, the querier retries fetching series from missing blocks up to 3 times. If the consistency check fails once all retries have been exhausted, the query execution fails. The following metrics have been added: #2593 #2630 #2689 #2695 cortex_querier_blocks_consistency_checks_total cortex_querier_blocks_consistency_checks_failed_total cortex_querier_storegateway_refetches_per_query [ENHANCEMENT] Delete requests can now be canceled #2555 [ENHANCEMENT] Table manager can now provision tables for delete store #2546 [BUGFIX] Ruler: Ensure temporary rule files with special characters are properly mapped and cleaned up. #2506 [BUGFIX] Fixes #2411, Ensure requests are properly routed to the prometheus api embedded in the query if -server.path-prefix is set. #2372 [BUGFIX] Experimental TSDB: fixed chunk data corruption when querying back series using the experimental blocks storage. #2400 [BUGFIX] Fixed collection of tracing spans from Thanos components used internally. #2655 [BUGFIX] Experimental TSDB: fixed memory leak in ingesters. #2586 [BUGFIX] QueryFrontend: fixed a situation where HTTP error is ignored and an incorrect status code is set. #2590 [BUGFIX] Ingester: Fix an ingester starting up in the JOINING state and staying there forever. #2565 [BUGFIX] QueryFrontend: fixed a panic (integer divide by zero) in the query-frontend. The query-frontend now requires the -querier.default-evaluation-interval config to be set to the same value of the querier. #2614 [BUGFIX] Experimental TSDB: when the querier receives a /series request with a time range older than the data stored in the ingester, it now ignores the requested time range and returns known series anyway instead of returning an empty response. This aligns the behaviour with the chunks storage. #2617 [BUGFIX] Cassandra: fixed an edge case leading to an invalid CQL query when querying the index on a Cassandra store. #2639 [BUGFIX] Ingester: increment series per metric when recovering from WAL or transfer. #2674 [BUGFIX] Fixed wrong number of arguments for 'mget' command Redis error when a query has no chunks to lookup from storage. #2700 #2796 [BUGFIX] Ingester: Automatically remove old tmp checkpoints, fixing a potential disk space leak after an ingester crashes. #2726 1.1.0 / 2020-05-21 This release brings the usual mix of bugfixes and improvements. The biggest change is that WAL support for chunks is now considered to be production-ready!\nPlease make sure to review renamed metrics, and update your dashboards and alerts accordingly.\n[CHANGE] Added v1 API routes documented in #2327. #2372 Added -http.alertmanager-http-prefix flag which allows the configuration of the path where the Alertmanager API and UI can be reached. The default is set to /alertmanager. Added -http.prometheus-http-prefix flag which allows the configuration of the path where the Prometheus API and UI can be reached. The default is set to /prometheus. Updated the index hosted at the root prefix to point to the updated routes. Legacy routes hardcoded with the /api/prom prefix now respect the -http.prefix flag. [CHANGE] The metrics cortex_distributor_ingester_appends_total and distributor_ingester_append_failures_total now include a type label to differentiate between samples and metadata. #2336 [CHANGE] The metrics for number of chunks and bytes flushed to the chunk store are renamed. Note that previous metrics were counted pre-deduplication, while new metrics are counted after deduplication. #2463 cortex_ingester_chunks_stored_total \u003e cortex_chunk_store_stored_chunks_total cortex_ingester_chunk_stored_bytes_total \u003e cortex_chunk_store_stored_chunk_bytes_total [CHANGE] Experimental TSDB: renamed blocks meta fetcher metrics: #2375 cortex_querier_bucket_store_blocks_meta_syncs_total \u003e cortex_querier_blocks_meta_syncs_total cortex_querier_bucket_store_blocks_meta_sync_failures_total \u003e cortex_querier_blocks_meta_sync_failures_total cortex_querier_bucket_store_blocks_meta_sync_duration_seconds \u003e cortex_querier_blocks_meta_sync_duration_seconds cortex_querier_bucket_store_blocks_meta_sync_consistency_delay_seconds \u003e cortex_querier_blocks_meta_sync_consistency_delay_seconds [CHANGE] Experimental TSDB: Modified default values for compactor.deletion-delay option from 48h to 12h and -experimental.tsdb.bucket-store.ignore-deletion-marks-delay from 24h to 6h. #2414 [CHANGE] WAL: Default value of -ingester.checkpoint-enabled changed to true. #2416 [CHANGE] trace_id field in log files has been renamed to traceID. #2518 [CHANGE] Slow query log has a different output now. Previously used url field has been replaced with host and path, and query parameters are logged as individual log fields with qs_ prefix. #2520 [CHANGE] WAL: WAL and checkpoint compression is now disabled. #2436 [CHANGE] Update in dependency go-kit/kit from v0.9.0 to v0.10.0. HTML escaping disabled in JSON Logger. #2535 [CHANGE] Experimental TSDB: Removed cortex_\u003cservice\u003e_ prefix from Thanos objstore metrics and added component label to distinguish which Cortex component is doing API calls to the object storage when running in single-binary mode: #2568 cortex_\u003cservice\u003e_thanos_objstore_bucket_operations_total renamed to thanos_objstore_bucket_operations_total{component=\"\u003cname\u003e\"} cortex_\u003cservice\u003e_thanos_objstore_bucket_operation_failures_total renamed to thanos_objstore_bucket_operation_failures_total{component=\"\u003cname\u003e\"} cortex_\u003cservice\u003e_thanos_objstore_bucket_operation_duration_seconds renamed to thanos_objstore_bucket_operation_duration_seconds{component=\"\u003cname\u003e\"} cortex_\u003cservice\u003e_thanos_objstore_bucket_last_successful_upload_time renamed to thanos_objstore_bucket_last_successful_upload_time{component=\"\u003cname\u003e\"} [CHANGE] FIFO cache: The -\u003cprefix\u003e.fifocache.size CLI flag has been renamed to -\u003cprefix\u003e.fifocache.max-size-items as well as its YAML config option size renamed to max_size_items. #2319 [FEATURE] Ruler: The -ruler.evaluation-delay flag was added to allow users to configure a default evaluation delay for all rules in cortex. The default value is 0 which is the current behavior. #2423 [FEATURE] Experimental: Added a new object storage client for OpenStack Swift. #2440 [FEATURE] TLS config options added to the Server. #2535 [FEATURE] Experimental: Added support for /api/v1/metadata Prometheus-based endpoint. #2549 [FEATURE] Add ability to limit concurrent queries to Cassandra with -cassandra.query-concurrency flag. #2562 [FEATURE] Experimental TSDB: Introduced store-gateway service used by the experimental blocks storage to load and query blocks. The store-gateway optionally supports blocks sharding and replication via a dedicated hash ring, configurable via -experimental.store-gateway.sharding-enabled and -experimental.store-gateway.sharding-ring.* flags. The following metrics have been added: #2433 #2458 #2469 #2523 cortex_querier_storegateway_instances_hit_per_query [ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing cortex_discarded_samples_total metric. #2370 [ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. #2383 [ENHANCEMENT] Experimental TSDB: Added cortex_querier_blocks_meta_synced, which reflects current state of synced blocks over all tenants. #2392 [ENHANCEMENT] Added cortex_distributor_latest_seen_sample_timestamp_seconds metric to see how far behind Prometheus servers are in sending data. #2371 [ENHANCEMENT] FIFO cache to support eviction based on memory usage. Added -\u003cprefix\u003e.fifocache.max-size-bytes CLI flag and YAML config option max_size_bytes to specify memory limit of the cache. #2319, #2527 [ENHANCEMENT] Added -querier.worker-match-max-concurrent. Force worker concurrency to match the -querier.max-concurrent option. Overrides -querier.worker-parallelism. #2456 [ENHANCEMENT] Added the following metrics for monitoring delete requests: #2445 cortex_purger_delete_requests_received_total: Number of delete requests received per user. cortex_purger_delete_requests_processed_total: Number of delete requests processed per user. cortex_purger_delete_requests_chunks_selected_total: Number of chunks selected while building delete plans per user. cortex_purger_delete_requests_processing_failures_total: Number of delete requests processing failures per user. [ENHANCEMENT] Single Binary: Added query-frontend to the single binary. Single binary users will now benefit from various query-frontend features. Primarily: sharding, parallelization, load shedding, additional caching (if configured), and query retries. #2437 [ENHANCEMENT] Allow 1w (where w denotes week) and 1y (where y denotes year) when setting -store.cache-lookups-older-than and -store.max-look-back-period. #2454 [ENHANCEMENT] Optimize index queries for matchers using “a|b|c”-type regex. #2446 #2475 [ENHANCEMENT] Added per tenant metrics for queries and chunks and bytes read from chunk store: #2463 cortex_chunk_store_fetched_chunks_total and cortex_chunk_store_fetched_chunk_bytes_total cortex_query_frontend_queries_total (per tenant queries counted by the frontend) [ENHANCEMENT] WAL: New metrics cortex_ingester_wal_logged_bytes_total and cortex_ingester_checkpoint_logged_bytes_total added to track total bytes logged to disk for WAL and checkpoints. #2497 [ENHANCEMENT] Add de-duplicated chunks counter cortex_chunk_store_deduped_chunks_total which counts every chunk not sent to the store because it was already sent by another replica. #2485 [ENHANCEMENT] Query-frontend now also logs the POST data of long queries. #2481 [ENHANCEMENT] WAL: Ingester WAL records now have type header and the custom WAL records have been replaced by Prometheus TSDB’s WAL records. Old records will not be supported from 1.3 onwards. Note: once this is deployed, you cannot downgrade without data loss. #2436 [ENHANCEMENT] Redis Cache: Added idle_timeout, wait_on_pool_exhaustion and max_conn_lifetime options to redis cache configuration. #2550 [ENHANCEMENT] WAL: the experimental tag has been removed on the WAL in ingesters. #2560 [ENHANCEMENT] Use newer AWS API for paginated queries - removes ‘Deprecated’ message from logfiles. #2452 [ENHANCEMENT] Experimental memberlist: Add retry with backoff on memberlist join other members. #2705 [ENHANCEMENT] Experimental TSDB: when the store-gateway sharding is enabled, unhealthy store-gateway instances are automatically removed from the ring after 10 consecutive -experimental.store-gateway.sharding-ring.heartbeat-timeout periods. #2526 [BUGFIX] Ruler: Ensure temporary rule files with special characters are properly mapped and cleaned up. #2506 [BUGFIX] Ensure requests are properly routed to the prometheus api embedded in the query if -server.path-prefix is set. Fixes #2411. #2372 [BUGFIX] Experimental TSDB: Fixed chunk data corruption when querying back series using the experimental blocks storage. #2400 [BUGFIX] Cassandra Storage: Fix endpoint TLS host verification. #2109 [BUGFIX] Experimental TSDB: Fixed response status code from 422 to 500 when an error occurs while iterating chunks with the experimental blocks storage. #2402 [BUGFIX] Ring: Fixed a situation where upgrading from pre-1.0 cortex with a rolling strategy caused new 1.0 ingesters to lose their zone value in the ring until manually forced to re-register. #2404 [BUGFIX] Distributor: /all_user_stats now show API and Rule Ingest Rate correctly. #2457 [BUGFIX] Fixed version, revision and branch labels exported by the cortex_build_info metric. #2468 [BUGFIX] QueryFrontend: fixed a situation where span context missed when downstream_url is used. #2539 [BUGFIX] Querier: Fixed a situation where querier would crash because of an unresponsive frontend instance. #2569 1.0.1 / 2020-04-23 [BUGFIX] Fix gaps when querying ingesters with replication factor = 3 and 2 ingesters in the cluster. #2503 1.0.0 / 2020-04-02 This is the first major release of Cortex. We made a lot of breaking changes in this release which have been detailed below. Please also see the stability guarantees we provide as part of a major release: https://cortexmetrics.io/docs/configuration/v1guarantees/\n[CHANGE] Remove the following deprecated flags: #2339\n-metrics.error-rate-query (use -metrics.write-throttle-query instead). -store.cardinality-cache-size (use -store.index-cache-read.enable-fifocache and -store.index-cache-read.fifocache.size instead). -store.cardinality-cache-validity (use -store.index-cache-read.enable-fifocache and -store.index-cache-read.fifocache.duration instead). -distributor.limiter-reload-period (flag unused) -ingester.claim-on-rollout (flag unused) -ingester.normalise-tokens (flag unused) [CHANGE] Renamed YAML file options to be more consistent. See full config file changes below. #2273\n[CHANGE] AWS based autoscaling has been removed. You can only use metrics based autoscaling now. -applicationautoscaling.url has been removed. See https://cortexmetrics.io/docs/production/aws/#dynamodb-capacity-provisioning on how to migrate. #2328\n[CHANGE] Renamed the memcache.write-back-goroutines and memcache.write-back-buffer flags to background.write-back-concurrency and background.write-back-buffer. This affects the following flags: #2241\n-frontend.memcache.write-back-buffer –\u003e -frontend.background.write-back-buffer -frontend.memcache.write-back-goroutines –\u003e -frontend.background.write-back-concurrency -store.index-cache-read.memcache.write-back-buffer –\u003e -store.index-cache-read.background.write-back-buffer -store.index-cache-read.memcache.write-back-goroutines –\u003e -store.index-cache-read.background.write-back-concurrency -store.index-cache-write.memcache.write-back-buffer –\u003e -store.index-cache-write.background.write-back-buffer -store.index-cache-write.memcache.write-back-goroutines –\u003e -store.index-cache-write.background.write-back-concurrency -memcache.write-back-buffer –\u003e -store.chunks-cache.background.write-back-buffer. Note the next change log for the difference. -memcache.write-back-goroutines –\u003e -store.chunks-cache.background.write-back-concurrency. Note the next change log for the difference. [CHANGE] Renamed the chunk cache flags to have store.chunks-cache. as prefix. This means the following flags have been changed: #2241\n-cache.enable-fifocache –\u003e -store.chunks-cache.cache.enable-fifocache -default-validity –\u003e -store.chunks-cache.default-validity -fifocache.duration –\u003e -store.chunks-cache.fifocache.duration -fifocache.size –\u003e -store.chunks-cache.fifocache.size -memcache.write-back-buffer –\u003e -store.chunks-cache.background.write-back-buffer. Note the previous change log for the difference. -memcache.write-back-goroutines –\u003e -store.chunks-cache.background.write-back-concurrency. Note the previous change log for the difference. -memcached.batchsize –\u003e -store.chunks-cache.memcached.batchsize -memcached.consistent-hash –\u003e -store.chunks-cache.memcached.consistent-hash -memcached.expiration –\u003e -store.chunks-cache.memcached.expiration -memcached.hostname –\u003e -store.chunks-cache.memcached.hostname -memcached.max-idle-conns –\u003e -store.chunks-cache.memcached.max-idle-conns -memcached.parallelism –\u003e -store.chunks-cache.memcached.parallelism -memcached.service –\u003e -store.chunks-cache.memcached.service -memcached.timeout –\u003e -store.chunks-cache.memcached.timeout -memcached.update-interval –\u003e -store.chunks-cache.memcached.update-interval -redis.enable-tls –\u003e -store.chunks-cache.redis.enable-tls -redis.endpoint –\u003e -store.chunks-cache.redis.endpoint -redis.expiration –\u003e -store.chunks-cache.redis.expiration -redis.max-active-conns –\u003e -store.chunks-cache.redis.max-active-conns -redis.max-idle-conns –\u003e -store.chunks-cache.redis.max-idle-conns -redis.password –\u003e -store.chunks-cache.redis.password -redis.timeout –\u003e -store.chunks-cache.redis.timeout [CHANGE] Rename the -store.chunk-cache-stubs to -store.chunks-cache.cache-stubs to be more inline with above. #2241\n[CHANGE] Change prefix of flags -dynamodb.periodic-table.* to -table-manager.index-table.*. #2359\n[CHANGE] Change prefix of flags -dynamodb.chunk-table.* to -table-manager.chunk-table.*. #2359\n[CHANGE] Change the following flags: #2359\n-dynamodb.poll-interval –\u003e -table-manager.poll-interval -dynamodb.periodic-table.grace-period –\u003e -table-manager.periodic-table.grace-period [CHANGE] Renamed the following flags: #2273\n-dynamodb.chunk.gang.size –\u003e -dynamodb.chunk-gang-size -dynamodb.chunk.get.max.parallelism –\u003e -dynamodb.chunk-get-max-parallelism [CHANGE] Don’t support mixed time units anymore for duration. For example, 168h5m0s doesn’t work anymore, please use just one unit (s|m|h|d|w|y). #2252\n[CHANGE] Utilize separate protos for rule state and storage. Experimental ruler API will not be functional until the rollout is complete. #2226\n[CHANGE] Frontend worker in querier now starts after all Querier module dependencies are started. This fixes issue where frontend worker started to send queries to querier before it was ready to serve them (mostly visible when using experimental blocks storage). #2246\n[CHANGE] Lifecycler component now enters Failed state on errors, and doesn’t exit the process. (Important if you’re vendoring Cortex and use Lifecycler) #2251\n[CHANGE] /ready handler now returns 200 instead of 204. #2330\n[CHANGE] Better defaults for the following options: #2344\n-\u003cprefix\u003e.consul.consistent-reads: Old default: true, new default: false. This reduces the load on Consul. -\u003cprefix\u003e.consul.watch-rate-limit: Old default: 0, new default: 1. This rate limits the reads to 1 per second. Which is good enough for ring watches. -distributor.health-check-ingesters: Old default: false, new default: true. -ingester.max-stale-chunk-idle: Old default: 0, new default: 2m. This lets us expire series that we know are stale early. -ingester.spread-flushes: Old default: false, new default: true. This allows to better de-duplicate data and use less space. -ingester.chunk-age-jitter: Old default: 20mins, new default: 0. This is to enable the -ingester.spread-flushes to true. -\u003cprefix\u003e.memcached.batchsize: Old default: 0, new default: 1024. This allows batching of requests and keeps the concurrent requests low. -\u003cprefix\u003e.memcached.consistent-hash: Old default: false, new default: true. This allows for better cache hits when the memcaches are scaled up and down. -querier.batch-iterators: Old default: false, new default: true. -querier.ingester-streaming: Old default: false, new default: true. [CHANGE] Experimental TSDB: Added -experimental.tsdb.bucket-store.postings-cache-compression-enabled to enable postings compression when storing to cache. #2335\n[CHANGE] Experimental TSDB: Added -compactor.deletion-delay, which is time before a block marked for deletion is deleted from bucket. If not 0, blocks will be marked for deletion and compactor component will delete blocks marked for deletion from the bucket. If delete-delay is 0, blocks will be deleted straight away. Note that deleting blocks immediately can cause query failures, if store gateway / querier still has the block loaded, or compactor is ignoring the deletion because it’s compacting the block at the same time. Default value is 48h. #2335\n[CHANGE] Experimental TSDB: Added -experimental.tsdb.bucket-store.index-cache.postings-compression-enabled, to set duration after which the blocks marked for deletion will be filtered out while fetching blocks used for querying. This option allows querier to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet. Default is 24h, half of the default value for -compactor.deletion-delay. #2335\n[CHANGE] Experimental TSDB: Added -experimental.tsdb.bucket-store.index-cache.memcached.max-item-size to control maximum size of item that is stored to memcached. Defaults to 1 MiB. #2335\n[FEATURE] Added experimental storage API to the ruler service that is enabled when the -experimental.ruler.enable-api is set to true #2269\n-ruler.storage.type flag now allows s3,gcs, and azure values -ruler.storage.(s3|gcs|azure) flags exist to allow the configuration of object clients set for rule storage [CHANGE] Renamed table manager metrics. #2307 #2359\ncortex_dynamo_sync_tables_seconds -\u003e cortex_table_manager_sync_duration_seconds cortex_dynamo_table_capacity_units -\u003e cortex_table_capacity_units [FEATURE] Flusher target to flush the WAL. #2075\n-flusher.wal-dir for the WAL directory to recover from. -flusher.concurrent-flushes for number of concurrent flushes. -flusher.flush-op-timeout is duration after which a flush should timeout. [FEATURE] Ingesters can now have an optional availability zone set, to ensure metric replication is distributed across zones. This is set via the -ingester.availability-zone flag or the availability_zone field in the config file. #2317\n[ENHANCEMENT] Better reuse of connections to DynamoDB and S3. #2268\n[ENHANCEMENT] Reduce number of goroutines used while executing a single index query. #2280\n[ENHANCEMENT] Experimental TSDB: Add support for local filesystem backend. #2245\n[ENHANCEMENT] Experimental TSDB: Added memcached support for the TSDB index cache. #2290\n[ENHANCEMENT] Experimental TSDB: Removed gRPC server to communicate between querier and BucketStore. #2324\n[ENHANCEMENT] Allow 1w (where w denotes week) and 1y (where y denotes year) when setting table period and retention. #2252\n[ENHANCEMENT] Added FIFO cache metrics for current number of entries and memory usage. #2270\n[ENHANCEMENT] Output all config fields to /config API, including those with empty value. #2209\n[ENHANCEMENT] Add “missing_metric_name” and “metric_name_invalid” reasons to cortex_discarded_samples_total metric. #2346\n[ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing cortex_discarded_samples_total metric. #2370\n[BUGFIX] Ensure user state metrics are updated if a transfer fails. #2338\n[BUGFIX] Fixed etcd client keepalive settings. #2278\n[BUGFIX] Register the metrics of the WAL. #2295\n[BUXFIX] Experimental TSDB: fixed error handling when ingesting out of bound samples. #2342\nKnown issues This experimental blocks storage in Cortex 1.0.0 has a bug which may lead to the error cannot iterate chunk for series when running queries. This bug has been fixed in #2400. If you’re running the experimental blocks storage, please build Cortex from master. Config file breaking changes In this section you can find a config file diff showing the breaking changes introduced in Cortex. You can also find the full configuration file reference doc in the website.\n### ingester_config # Period with which to attempt to flush chunks. # CLI flag: -ingester.flush-period -[flushcheckperiod: \u003cduration\u003e | default = 1m0s] +[flush_period: \u003cduration\u003e | default = 1m0s] # Period chunks will remain in memory after flushing. # CLI flag: -ingester.retain-period -[retainperiod: \u003cduration\u003e | default = 5m0s] +[retain_period: \u003cduration\u003e | default = 5m0s] # Maximum chunk idle time before flushing. # CLI flag: -ingester.max-chunk-idle -[maxchunkidle: \u003cduration\u003e | default = 5m0s] +[max_chunk_idle_time: \u003cduration\u003e | default = 5m0s] # Maximum chunk idle time for chunks terminating in stale markers before # flushing. 0 disables it and a stale series is not flushed until the # max-chunk-idle timeout is reached. # CLI flag: -ingester.max-stale-chunk-idle -[maxstalechunkidle: \u003cduration\u003e | default = 0s] +[max_stale_chunk_idle_time: \u003cduration\u003e | default = 2m0s] # Timeout for individual flush operations. # CLI flag: -ingester.flush-op-timeout -[flushoptimeout: \u003cduration\u003e | default = 1m0s] +[flush_op_timeout: \u003cduration\u003e | default = 1m0s] # Maximum chunk age before flushing. # CLI flag: -ingester.max-chunk-age -[maxchunkage: \u003cduration\u003e | default = 12h0m0s] +[max_chunk_age: \u003cduration\u003e | default = 12h0m0s] -# Range of time to subtract from MaxChunkAge to spread out flushes +# Range of time to subtract from -ingester.max-chunk-age to spread out flushes # CLI flag: -ingester.chunk-age-jitter -[chunkagejitter: \u003cduration\u003e | default = 20m0s] +[chunk_age_jitter: \u003cduration\u003e | default = 0] # Number of concurrent goroutines flushing to dynamodb. # CLI flag: -ingester.concurrent-flushes -[concurrentflushes: \u003cint\u003e | default = 50] +[concurrent_flushes: \u003cint\u003e | default = 50] -# If true, spread series flushes across the whole period of MaxChunkAge +# If true, spread series flushes across the whole period of +# -ingester.max-chunk-age. # CLI flag: -ingester.spread-flushes -[spreadflushes: \u003cboolean\u003e | default = false] +[spread_flushes: \u003cboolean\u003e | default = true] # Period with which to update the per-user ingestion rates. # CLI flag: -ingester.rate-update-period -[rateupdateperiod: \u003cduration\u003e | default = 15s] +[rate_update_period: \u003cduration\u003e | default = 15s] ### querier_config # The maximum number of concurrent queries. # CLI flag: -querier.max-concurrent -[maxconcurrent: \u003cint\u003e | default = 20] +[max_concurrent: \u003cint\u003e | default = 20] # Use batch iterators to execute query, as opposed to fully materialising the # series in memory. Takes precedent over the -querier.iterators flag. # CLI flag: -querier.batch-iterators -[batchiterators: \u003cboolean\u003e | default = false] +[batch_iterators: \u003cboolean\u003e | default = true] # Use streaming RPCs to query ingester. # CLI flag: -querier.ingester-streaming -[ingesterstreaming: \u003cboolean\u003e | default = false] +[ingester_streaming: \u003cboolean\u003e | default = true] # Maximum number of samples a single query can load into memory. # CLI flag: -querier.max-samples -[maxsamples: \u003cint\u003e | default = 50000000] +[max_samples: \u003cint\u003e | default = 50000000] # The default evaluation interval or step size for subqueries. # CLI flag: -querier.default-evaluation-interval -[defaultevaluationinterval: \u003cduration\u003e | default = 1m0s] +[default_evaluation_interval: \u003cduration\u003e | default = 1m0s] ### query_frontend_config # URL of downstream Prometheus. # CLI flag: -frontend.downstream-url -[downstream: \u003cstring\u003e | default = \"\"] +[downstream_url: \u003cstring\u003e | default = \"\"] ### ruler_config # URL of alerts return path. # CLI flag: -ruler.external.url -[externalurl: \u003curl\u003e | default = ] +[external_url: \u003curl\u003e | default = ] # How frequently to evaluate rules # CLI flag: -ruler.evaluation-interval -[evaluationinterval: \u003cduration\u003e | default = 1m0s] +[evaluation_interval: \u003cduration\u003e | default = 1m0s] # How frequently to poll for rule changes # CLI flag: -ruler.poll-interval -[pollinterval: \u003cduration\u003e | default = 1m0s] +[poll_interval: \u003cduration\u003e | default = 1m0s] -storeconfig: +storage: # file path to store temporary rule files for the prometheus rule managers # CLI flag: -ruler.rule-path -[rulepath: \u003cstring\u003e | default = \"/rules\"] +[rule_path: \u003cstring\u003e | default = \"/rules\"] # URL of the Alertmanager to send notifications to. # CLI flag: -ruler.alertmanager-url -[alertmanagerurl: \u003curl\u003e | default = ] +[alertmanager_url: \u003curl\u003e | default = ] # Use DNS SRV records to discover alertmanager hosts. # CLI flag: -ruler.alertmanager-discovery -[alertmanagerdiscovery: \u003cboolean\u003e | default = false] +[enable_alertmanager_discovery: \u003cboolean\u003e | default = false] # How long to wait between refreshing alertmanager hosts. # CLI flag: -ruler.alertmanager-refresh-interval -[alertmanagerrefreshinterval: \u003cduration\u003e | default = 1m0s] +[alertmanager_refresh_interval: \u003cduration\u003e | default = 1m0s] # If enabled requests to alertmanager will utilize the V2 API. # CLI flag: -ruler.alertmanager-use-v2 -[alertmanangerenablev2api: \u003cboolean\u003e | default = false] +[enable_alertmanager_v2: \u003cboolean\u003e | default = false] # Capacity of the queue for notifications to be sent to the Alertmanager. # CLI flag: -ruler.notification-queue-capacity -[notificationqueuecapacity: \u003cint\u003e | default = 10000] +[notification_queue_capacity: \u003cint\u003e | default = 10000] # HTTP timeout duration when sending notifications to the Alertmanager. # CLI flag: -ruler.notification-timeout -[notificationtimeout: \u003cduration\u003e | default = 10s] +[notification_timeout: \u003cduration\u003e | default = 10s] # Distribute rule evaluation using ring backend # CLI flag: -ruler.enable-sharding -[enablesharding: \u003cboolean\u003e | default = false] +[enable_sharding: \u003cboolean\u003e | default = false] # Time to spend searching for a pending ruler when shutting down. # CLI flag: -ruler.search-pending-for -[searchpendingfor: \u003cduration\u003e | default = 5m0s] +[search_pending_for: \u003cduration\u003e | default = 5m0s] # Period with which to attempt to flush rule groups. # CLI flag: -ruler.flush-period -[flushcheckperiod: \u003cduration\u003e | default = 1m0s] +[flush_period: \u003cduration\u003e | default = 1m0s] ### alertmanager_config # Base path for data storage. # CLI flag: -alertmanager.storage.path -[datadir: \u003cstring\u003e | default = \"data/\"] +[data_dir: \u003cstring\u003e | default = \"data/\"] # will be used to prefix all HTTP endpoints served by Alertmanager. If omitted, # relevant URL components will be derived automatically. # CLI flag: -alertmanager.web.external-url -[externalurl: \u003curl\u003e | default = ] +[external_url: \u003curl\u003e | default = ] # How frequently to poll Cortex configs # CLI flag: -alertmanager.configs.poll-interval -[pollinterval: \u003cduration\u003e | default = 15s] +[poll_interval: \u003cduration\u003e | default = 15s] # Listen address for cluster. # CLI flag: -cluster.listen-address -[clusterbindaddr: \u003cstring\u003e | default = \"0.0.0.0:9094\"] +[cluster_bind_address: \u003cstring\u003e | default = \"0.0.0.0:9094\"] # Explicit address to advertise in cluster. # CLI flag: -cluster.advertise-address -[clusteradvertiseaddr: \u003cstring\u003e | default = \"\"] +[cluster_advertise_address: \u003cstring\u003e | default = \"\"] # Time to wait between peers to send notifications. # CLI flag: -cluster.peer-timeout -[peertimeout: \u003cduration\u003e | default = 15s] +[peer_timeout: \u003cduration\u003e | default = 15s] # Filename of fallback config to use if none specified for instance. # CLI flag: -alertmanager.configs.fallback -[fallbackconfigfile: \u003cstring\u003e | default = \"\"] +[fallback_config_file: \u003cstring\u003e | default = \"\"] # Root of URL to generate if config is http://internal.monitor # CLI flag: -alertmanager.configs.auto-webhook-root -[autowebhookroot: \u003cstring\u003e | default = \"\"] +[auto_webhook_root: \u003cstring\u003e | default = \"\"] ### table_manager_config -store: +storage: -# How frequently to poll DynamoDB to learn our capacity. -# CLI flag: -dynamodb.poll-interval -[dynamodb_poll_interval: \u003cduration\u003e | default = 2m0s] +# How frequently to poll backend to learn our capacity. +# CLI flag: -table-manager.poll-interval +[poll_interval: \u003cduration\u003e | default = 2m0s] -# DynamoDB periodic tables grace period (duration which table will be -# created/deleted before/after it's needed). -# CLI flag: -dynamodb.periodic-table.grace-period +# Periodic tables grace period (duration which table will be created/deleted +# before/after it's needed). +# CLI flag: -table-manager.periodic-table.grace-period [creation_grace_period: \u003cduration\u003e | default = 10m0s] index_tables_provisioning: # Enables on demand throughput provisioning for the storage provider (if - # supported). Applies only to tables which are not autoscaled - # CLI flag: -dynamodb.periodic-table.enable-ondemand-throughput-mode - [provisioned_throughput_on_demand_mode: \u003cboolean\u003e | default = false] + # supported). Applies only to tables which are not autoscaled. Supported by + # DynamoDB + # CLI flag: -table-manager.index-table.enable-ondemand-throughput-mode + [enable_ondemand_throughput_mode: \u003cboolean\u003e | default = false] # Enables on demand throughput provisioning for the storage provider (if - # supported). Applies only to tables which are not autoscaled - # CLI flag: -dynamodb.periodic-table.inactive-enable-ondemand-throughput-mode - [inactive_throughput_on_demand_mode: \u003cboolean\u003e | default = false] + # supported). Applies only to tables which are not autoscaled. Supported by + # DynamoDB + # CLI flag: -table-manager.index-table.inactive-enable-ondemand-throughput-mode + [enable_inactive_throughput_on_demand_mode: \u003cboolean\u003e | default = false] chunk_tables_provisioning: # Enables on demand throughput provisioning for the storage provider (if - # supported). Applies only to tables which are not autoscaled - # CLI flag: -dynamodb.chunk-table.enable-ondemand-throughput-mode - [provisioned_throughput_on_demand_mode: \u003cboolean\u003e | default = false] + # supported). Applies only to tables which are not autoscaled. Supported by + # DynamoDB + # CLI flag: -table-manager.chunk-table.enable-ondemand-throughput-mode + [enable_ondemand_throughput_mode: \u003cboolean\u003e | default = false] ### storage_config aws: - dynamodbconfig: + dynamodb: # DynamoDB endpoint URL with escaped Key and Secret encoded. If only region # is specified as a host, proper endpoint will be deduced. Use # inmemory:///\u003ctable-name\u003e to use a mock in-memory implementation. # CLI flag: -dynamodb.url - [dynamodb: \u003curl\u003e | default = ] + [dynamodb_url: \u003curl\u003e | default = ] # DynamoDB table management requests per second limit. # CLI flag: -dynamodb.api-limit - [apilimit: \u003cfloat\u003e | default = 2] + [api_limit: \u003cfloat\u003e | default = 2] # DynamoDB rate cap to back off when throttled. # CLI flag: -dynamodb.throttle-limit - [throttlelimit: \u003cfloat\u003e | default = 10] + [throttle_limit: \u003cfloat\u003e | default = 10] - - # ApplicationAutoscaling endpoint URL with escaped Key and Secret encoded. - # CLI flag: -applicationautoscaling.url - [applicationautoscaling: \u003curl\u003e | default = ] # Queue length above which we will scale up capacity # CLI flag: -metrics.target-queue-length - [targetqueuelen: \u003cint\u003e | default = 100000] + [target_queue_length: \u003cint\u003e | default = 100000] # Scale up capacity by this multiple # CLI flag: -metrics.scale-up-factor - [scaleupfactor: \u003cfloat\u003e | default = 1.3] + [scale_up_factor: \u003cfloat\u003e | default = 1.3] # Ignore throttling below this level (rate per second) # CLI flag: -metrics.ignore-throttle-below - [minthrottling: \u003cfloat\u003e | default = 1] + [ignore_throttle_below: \u003cfloat\u003e | default = 1] # query to fetch ingester queue length # CLI flag: -metrics.queue-length-query - [queuelengthquery: \u003cstring\u003e | default = \"sum(avg_over_time(cortex_ingester_flush_queue_length{job=\\\"cortex/ingester\\\"}[2m]))\"] + [queue_length_query: \u003cstring\u003e | default = \"sum(avg_over_time(cortex_ingester_flush_queue_length{job=\\\"cortex/ingester\\\"}[2m]))\"] # query to fetch throttle rates per table # CLI flag: -metrics.write-throttle-query - [throttlequery: \u003cstring\u003e | default = \"sum(rate(cortex_dynamo_throttled_total{operation=\\\"DynamoDB.BatchWriteItem\\\"}[1m])) by (table) \u003e 0\"] + [write_throttle_query: \u003cstring\u003e | default = \"sum(rate(cortex_dynamo_throttled_total{operation=\\\"DynamoDB.BatchWriteItem\\\"}[1m])) by (table) \u003e 0\"] # query to fetch write capacity usage per table # CLI flag: -metrics.usage-query - [usagequery: \u003cstring\u003e | default = \"sum(rate(cortex_dynamo_consumed_capacity_total{operation=\\\"DynamoDB.BatchWriteItem\\\"}[15m])) by (table) \u003e 0\"] + [write_usage_query: \u003cstring\u003e | default = \"sum(rate(cortex_dynamo_consumed_capacity_total{operation=\\\"DynamoDB.BatchWriteItem\\\"}[15m])) by (table) \u003e 0\"] # query to fetch read capacity usage per table # CLI flag: -metrics.read-usage-query - [readusagequery: \u003cstring\u003e | default = \"sum(rate(cortex_dynamo_consumed_capacity_total{operation=\\\"DynamoDB.QueryPages\\\"}[1h])) by (table) \u003e 0\"] + [read_usage_query: \u003cstring\u003e | default = \"sum(rate(cortex_dynamo_consumed_capacity_total{operation=\\\"DynamoDB.QueryPages\\\"}[1h])) by (table) \u003e 0\"] # query to fetch read errors per table # CLI flag: -metrics.read-error-query - [readerrorquery: \u003cstring\u003e | default = \"sum(increase(cortex_dynamo_failures_total{operation=\\\"DynamoDB.QueryPages\\\",error=\\\"ProvisionedThroughputExceededException\\\"}[1m])) by (table) \u003e 0\"] + [read_error_query: \u003cstring\u003e | default = \"sum(increase(cortex_dynamo_failures_total{operation=\\\"DynamoDB.QueryPages\\\",error=\\\"ProvisionedThroughputExceededException\\\"}[1m])) by (table) \u003e 0\"] # Number of chunks to group together to parallelise fetches (zero to # disable) - # CLI flag: -dynamodb.chunk.gang.size - [chunkgangsize: \u003cint\u003e | default = 10] + # CLI flag: -dynamodb.chunk-gang-size + [chunk_gang_size: \u003cint\u003e | default = 10] # Max number of chunk-get operations to start in parallel - # CLI flag: -dynamodb.chunk.get.max.parallelism - [chunkgetmaxparallelism: \u003cint\u003e | default = 32] + # CLI flag: -dynamodb.chunk.get-max-parallelism + [chunk_get_max_parallelism: \u003cint\u003e | default = 32] backoff_config: # Minimum delay when backing off. # CLI flag: -bigtable.backoff-min-period - [minbackoff: \u003cduration\u003e | default = 100ms] + [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -bigtable.backoff-max-period - [maxbackoff: \u003cduration\u003e | default = 10s] + [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -bigtable.backoff-retries - [maxretries: \u003cint\u003e | default = 10] + [max_retries: \u003cint\u003e | default = 10] # If enabled, once a tables info is fetched, it is cached. # CLI flag: -bigtable.table-cache.enabled - [tablecacheenabled: \u003cboolean\u003e | default = true] + [table_cache_enabled: \u003cboolean\u003e | default = true] # Duration to cache tables before checking again. # CLI flag: -bigtable.table-cache.expiration - [tablecacheexpiration: \u003cduration\u003e | default = 30m0s] + [table_cache_expiration: \u003cduration\u003e | default = 30m0s] # Cache validity for active index entries. Should be no higher than # -ingester.max-chunk-idle. # CLI flag: -store.index-cache-validity -[indexcachevalidity: \u003cduration\u003e | default = 5m0s] +[index_cache_validity: \u003cduration\u003e | default = 5m0s] ### ingester_client_config grpc_client_config: backoff_config: # Minimum delay when backing off. # CLI flag: -ingester.client.backoff-min-period - [minbackoff: \u003cduration\u003e | default = 100ms] + [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -ingester.client.backoff-max-period - [maxbackoff: \u003cduration\u003e | default = 10s] + [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -ingester.client.backoff-retries - [maxretries: \u003cint\u003e | default = 10] + [max_retries: \u003cint\u003e | default = 10] ### frontend_worker_config -# Address of query frontend service. +# Address of query frontend service, in host:port format. # CLI flag: -querier.frontend-address -[address: \u003cstring\u003e | default = \"\"] +[frontend_address: \u003cstring\u003e | default = \"\"] # How often to query DNS. # CLI flag: -querier.dns-lookup-period -[dnslookupduration: \u003cduration\u003e | default = 10s] +[dns_lookup_duration: \u003cduration\u003e | default = 10s] grpc_client_config: backoff_config: # Minimum delay when backing off. # CLI flag: -querier.frontend-client.backoff-min-period - [minbackoff: \u003cduration\u003e | default = 100ms] + [min_period: \u003cduration\u003e | default = 100ms] # Maximum delay when backing off. # CLI flag: -querier.frontend-client.backoff-max-period - [maxbackoff: \u003cduration\u003e | default = 10s] + [max_period: \u003cduration\u003e | default = 10s] # Number of times to backoff and retry before failing. # CLI flag: -querier.frontend-client.backoff-retries - [maxretries: \u003cint\u003e | default = 10] + [max_retries: \u003cint\u003e | default = 10] ### consul_config # ACL Token used to interact with Consul. -# CLI flag: -\u003cprefix\u003e.consul.acltoken -[acltoken: \u003cstring\u003e | default = \"\"] +# CLI flag: -\u003cprefix\u003e.consul.acl-token +[acl_token: \u003cstring\u003e | default = \"\"] # HTTP timeout when talking to Consul # CLI flag: -\u003cprefix\u003e.consul.client-timeout -[httpclienttimeout: \u003cduration\u003e | default = 20s] +[http_client_timeout: \u003cduration\u003e | default = 20s] # Enable consistent reads to Consul. # CLI flag: -\u003cprefix\u003e.consul.consistent-reads -[consistentreads: \u003cboolean\u003e | default = true] +[consistent_reads: \u003cboolean\u003e | default = false] # Rate limit when watching key or prefix in Consul, in requests per second. 0 # disables the rate limit. # CLI flag: -\u003cprefix\u003e.consul.watch-rate-limit -[watchkeyratelimit: \u003cfloat\u003e | default = 0] +[watch_rate_limit: \u003cfloat\u003e | default = 1] # Burst size used in rate limit. Values less than 1 are treated as 1. # CLI flag: -\u003cprefix\u003e.consul.watch-burst-size -[watchkeyburstsize: \u003cint\u003e | default = 1] +[watch_burst_size: \u003cint\u003e | default = 1] ### configstore_config # URL of configs API server. # CLI flag: -\u003cprefix\u003e.configs.url -[configsapiurl: \u003curl\u003e | default = ] +[configs_api_url: \u003curl\u003e | default = ] # Timeout for requests to Weave Cloud configs service. # CLI flag: -\u003cprefix\u003e.configs.client-timeout -[clienttimeout: \u003cduration\u003e | default = 5s] +[client_timeout: \u003cduration\u003e | default = 5s] 0.7.0 / 2020-03-16 Cortex 0.7.0 is a major step forward the upcoming 1.0 release. In this release, we’ve got 164 contributions from 26 authors. Thanks to all contributors! ❤️\nPlease be aware that Cortex 0.7.0 introduces some breaking changes. You’re encouraged to read all the [CHANGE] entries below before upgrading your Cortex cluster. In particular:\nCleaned up some configuration options in preparation for the Cortex 1.0.0 release (see also the annotated config file breaking changes below): Removed CLI flags support to configure the schema (see how to migrate from flags to schema file) Renamed CLI flag -config-yaml to -schema-config-file Removed CLI flag -store.min-chunk-age in favor of -querier.query-store-after. The corresponding YAML config option ingestermaxquerylookback has been renamed to query_ingesters_within Deprecated CLI flag -frontend.cache-split-interval in favor of -querier.split-queries-by-interval Renamed the YAML config option defaul_validity to default_validity Removed the YAML config option config_store (in the alertmanager YAML config) in favor of store Removed the YAML config root block configdb in favor of configs. This change is also reflected in the following CLI flags renaming: -database.* -\u003e -configs.database.* -database.migrations -\u003e -configs.database.migrations-dir Removed the fluentd-based billing infrastructure including the CLI flags: -distributor.enable-billing -billing.max-buffered-events -billing.retry-delay -billing.ingester Removed support for using denormalised tokens in the ring. Before upgrading, make sure your Cortex cluster is already running v0.6.0 or an earlier version with -ingester.normalise-tokens=true Full changelog [CHANGE] Removed support for flags to configure schema. Further, the flag for specifying the config file (-config-yaml) has been deprecated. Please use -schema-config-file. See the Schema Configuration documentation for more details on how to configure the schema using the YAML file. #2221 [CHANGE] In the config file, the root level config_store config option has been moved to alertmanager \u003e store \u003e configdb. #2125 [CHANGE] Removed unnecessary frontend.cache-split-interval in favor of querier.split-queries-by-interval both to reduce configuration complexity and guarantee alignment of these two configs. Starting from now, -querier.cache-results may only be enabled in conjunction with -querier.split-queries-by-interval (previously the cache interval default was 24h so if you want to preserve the same behaviour you should set -querier.split-queries-by-interval=24h). #2040 [CHANGE] Renamed Configs configuration options. #2187 configuration options -database.* -\u003e -configs.database.* -database.migrations -\u003e -configs.database.migrations-dir config file configdb.uri: -\u003e configs.database.uri: configdb.migrationsdir: -\u003e configs.database.migrations_dir: configdb.passwordfile: -\u003e configs.database.password_file: [CHANGE] Moved -store.min-chunk-age to the Querier config as -querier.query-store-after, allowing the store to be skipped during query time if the metrics wouldn’t be found. The YAML config option ingestermaxquerylookback has been renamed to query_ingesters_within to match its CLI flag. #1893 [CHANGE] Renamed the cache configuration setting defaul_validity to default_validity. #2140 [CHANGE] Remove fluentd-based billing infrastructure and flags such as -distributor.enable-billing. #1491 [CHANGE] Removed remaining support for using denormalised tokens in the ring. If you’re still running ingesters with denormalised tokens (Cortex 0.4 or earlier, with -ingester.normalise-tokens=false), such ingesters will now be completely invisible to distributors and need to be either switched to Cortex 0.6.0 or later, or be configured to use normalised tokens. #2034 [CHANGE] The frontend http server will now send 502 in case of deadline exceeded and 499 if the user requested cancellation. #2156 [CHANGE] We now enforce queries to be up to -querier.max-query-into-future into the future (defaults to 10m). #1929 -store.min-chunk-age has been removed -querier.query-store-after has been added in it’s place. [CHANGE] Removed unused /validate_expr endpoint. #2152 [CHANGE] Updated Prometheus dependency to v2.16.0. This Prometheus version uses Active Query Tracker to limit concurrent queries. In order to keep -querier.max-concurrent working, Active Query Tracker is enabled by default, and is configured to store its data to active-query-tracker directory (relative to current directory when Cortex started). This can be changed by using -querier.active-query-tracker-dir option. Purpose of Active Query Tracker is to log queries that were running when Cortex crashes. This logging happens on next Cortex start. #2088 [CHANGE] Default to BigChunk encoding; may result in slightly higher disk usage if many timeseries have a constant value, but should generally result in fewer, bigger chunks. #2207 [CHANGE] WAL replays are now done while the rest of Cortex is starting, and more specifically, when HTTP server is running. This makes it possible to scrape metrics during WAL replays. Applies to both chunks and experimental blocks storage. #2222 [CHANGE] Cortex now has /ready probe for all services, not just ingester and querier as before. In single-binary mode, /ready reports 204 only if all components are running properly. #2166 [CHANGE] If you are vendoring Cortex and use its components in your project, be aware that many Cortex components no longer start automatically when they are created. You may want to review PR and attached document. #2166 [CHANGE] Experimental TSDB: the querier in-memory index cache used by the experimental blocks storage shifted from per-tenant to per-querier. The -experimental.tsdb.bucket-store.index-cache-size-bytes now configures the per-querier index cache max size instead of a per-tenant cache and its default has been increased to 1GB. #2189 [CHANGE] Experimental TSDB: TSDB head compaction interval and concurrency is now configurable (defaults to 1 min interval and 5 concurrent head compactions). New options: -experimental.tsdb.head-compaction-interval and -experimental.tsdb.head-compaction-concurrency. #2172 [CHANGE] Experimental TSDB: switched the blocks storage index header to the binary format. This change is expected to have no visible impact, except lower startup times and memory usage in the queriers. It’s possible to switch back to the old JSON format via the flag -experimental.tsdb.bucket-store.binary-index-header-enabled=false. #2223 [CHANGE] Experimental Memberlist KV store can now be used in single-binary Cortex. Attempts to use it previously would fail with panic. This change also breaks existing binary protocol used to exchange gossip messages, so this version will not be able to understand gossiped Ring when used in combination with the previous version of Cortex. Easiest way to upgrade is to shutdown old Cortex installation, and restart it with new version. Incremental rollout works too, but with reduced functionality until all components run the same version. #2016 [FEATURE] Added a read-only local alertmanager config store using files named corresponding to their tenant id. #2125 [FEATURE] Added flag -experimental.ruler.enable-api to enable the ruler api which implements the Prometheus API /api/v1/rules and /api/v1/alerts endpoints under the configured -http.prefix. #1999 [FEATURE] Added sharding support to compactor when using the experimental TSDB blocks storage. #2113 [FEATURE] Added ability to override YAML config file settings using environment variables. #2147 -config.expand-env [FEATURE] Added flags to disable Alertmanager notifications methods. #2187 -configs.notifications.disable-email -configs.notifications.disable-webhook [FEATURE] Add /config HTTP endpoint which exposes the current Cortex configuration as YAML. #2165 [FEATURE] Allow Prometheus remote write directly to ingesters. #1491 [FEATURE] Introduced new standalone service query-tee that can be used for testing purposes to send the same Prometheus query to multiple backends (ie. two Cortex clusters ingesting the same metrics) and compare the performances. #2203 [FEATURE] Fan out parallelizable queries to backend queriers concurrently. #1878 querier.parallelise-shardable-queries (bool) Requires a shard-compatible schema (v10+) This causes the number of traces to increase accordingly. The query-frontend now requires a schema config to determine how/when to shard queries, either from a file or from flags (i.e. by the config-yaml CLI flag). This is the same schema config the queriers consume. The schema is only required to use this option. It’s also advised to increase downstream concurrency controls as well: querier.max-outstanding-requests-per-tenant querier.max-query-parallelism querier.max-concurrent server.grpc-max-concurrent-streams (for both query-frontends and queriers) [FEATURE] Added user sub rings to distribute users to a subset of ingesters. #1947 -experimental.distributor.user-subring-size [FEATURE] Add flag -experimental.tsdb.stripe-size to expose TSDB stripe size option. #2185 [FEATURE] Experimental Delete Series: Added support for Deleting Series with Prometheus style API. Needs to be enabled first by setting -purger.enable to true. Deletion only supported when using boltdb and filesystem as index and object store respectively. Support for other stores to follow in separate PRs #2103 [ENHANCEMENT] Alertmanager: Expose Per-tenant alertmanager metrics #2124 [ENHANCEMENT] Add status label to cortex_alertmanager_configs metric to gauge the number of valid and invalid configs. #2125 [ENHANCEMENT] Cassandra Authentication: added the custom_authenticators config option that allows users to authenticate with cassandra clusters using password authenticators that are not approved by default in gocql #2093 [ENHANCEMENT] Cassandra Storage: added max_retries, retry_min_backoff and retry_max_backoff configuration options to enable retrying recoverable errors. #2054 [ENHANCEMENT] Allow to configure HTTP and gRPC server listen address, maximum number of simultaneous connections and connection keepalive settings. -server.http-listen-address -server.http-conn-limit -server.grpc-listen-address -server.grpc-conn-limit -server.grpc.keepalive.max-connection-idle -server.grpc.keepalive.max-connection-age -server.grpc.keepalive.max-connection-age-grace -server.grpc.keepalive.time -server.grpc.keepalive.timeout [ENHANCEMENT] PostgreSQL: Bump up github.com/lib/pq from v1.0.0 to v1.3.0 to support PostgreSQL SCRAM-SHA-256 authentication. #2097 [ENHANCEMENT] Cassandra Storage: User no longer need CREATE privilege on \u003call keyspaces\u003e if given keyspace exists. #2032 [ENHANCEMENT] Cassandra Storage: added password_file configuration options to enable reading Cassandra password from file. #2096 [ENHANCEMENT] Configs API: Allow GET/POST configs in YAML format. #2181 [ENHANCEMENT] Background cache writes are batched to improve parallelism and observability. #2135 [ENHANCEMENT] Add automatic repair for checkpoint and WAL. #2105 [ENHANCEMENT] Support lastEvaluation and evaluationTime in /api/v1/rules endpoints and make order of groups stable. #2196 [ENHANCEMENT] Skip expired requests in query-frontend scheduling. #2082 [ENHANCEMENT] Add ability to configure gRPC keepalive settings. #2066 [ENHANCEMENT] Experimental TSDB: Export TSDB Syncer metrics from Compactor component, they are prefixed with cortex_compactor_. #2023 [ENHANCEMENT] Experimental TSDB: Added dedicated flag -experimental.tsdb.bucket-store.tenant-sync-concurrency to configure the maximum number of concurrent tenants for which blocks are synched. #2026 [ENHANCEMENT] Experimental TSDB: Expose metrics for objstore operations (prefixed with cortex_\u003ccomponent\u003e_thanos_objstore_, component being one of ingester, querier and compactor). #2027 [ENHANCEMENT] Experimental TSDB: Added support for Azure Storage to be used for block storage, in addition to S3 and GCS. #2083 [ENHANCEMENT] Experimental TSDB: Reduced memory allocations in the ingesters when using the experimental blocks storage. #2057 [ENHANCEMENT] Experimental Memberlist KV: expose -memberlist.gossip-to-dead-nodes-time and -memberlist.dead-node-reclaim-time options to control how memberlist library handles dead nodes and name reuse. #2131 [BUGFIX] Alertmanager: fixed panic upon applying a new config, caused by duplicate metrics registration in the NewPipelineBuilder function. #211 [BUGFIX] Azure Blob ChunkStore: Fixed issue causing invalid chunk checksum errors. #2074 [BUGFIX] The gauge cortex_overrides_last_reload_successful is now only exported by components that use a RuntimeConfigManager. Previously, for components that do not initialize a RuntimeConfigManager (such as the compactor) the gauge was initialized with 0 (indicating error state) and then never updated, resulting in a false-negative permanent error state. #2092 [BUGFIX] Fixed WAL metric names, added the cortex_ prefix. [BUGFIX] Restored histogram cortex_configs_request_duration_seconds #2138 [BUGFIX] Fix wrong syntax for url in config-file-reference. #2148 [BUGFIX] Fixed some 5xx status code returned by the query-frontend when they should actually be 4xx. #2122 [BUGFIX] Fixed leaked goroutines in the querier. #2070 [BUGFIX] Experimental TSDB: fixed /all_user_stats and /api/prom/user_stats endpoints when using the experimental TSDB blocks storage. #2042 [BUGFIX] Experimental TSDB: fixed ruler to correctly work with the experimental TSDB blocks storage. #2101 Changes to denormalised tokens in the ring Cortex 0.4.0 is the last version that can write denormalised tokens. Cortex 0.5.0 and above always write normalised tokens.\nCortex 0.6.0 is the last version that can read denormalised tokens. Starting with Cortex 0.7.0 only normalised tokens are supported, and ingesters writing denormalised tokens to the ring (running Cortex 0.4.0 or earlier with -ingester.normalise-tokens=false) are ignored by distributors. Such ingesters should either switch to using normalised tokens, or be upgraded to Cortex 0.5.0 or later.\nKnown issues The gRPC streaming for ingesters doesn’t work when using the experimental TSDB blocks storage. Please do not enable -querier.ingester-streaming if you’re using the TSDB blocks storage. If you want to enable it, you can build Cortex from master given the issue has been fixed after Cortex 0.7 branch has been cut and the fix wasn’t included in the 0.7 because related to an experimental feature. Annotated config file breaking changes In this section you can find a config file diff showing the breaking changes introduced in Cortex 0.7. You can also find the full configuration file reference doc in the website.\n### Root level config # \"configdb\" has been moved to \"alertmanager \u003e store \u003e configdb\". -[configdb: \u003cconfigdb_config\u003e] # \"config_store\" has been renamed to \"configs\". -[config_store: \u003cconfigstore_config\u003e] +[configs: \u003cconfigs_config\u003e] ### `distributor_config` # The support to hook an external billing system has been removed. -[enable_billing: \u003cboolean\u003e | default = false] -billing: - [maxbufferedevents: \u003cint\u003e | default = 1024] - [retrydelay: \u003cduration\u003e | default = 500ms] - [ingesterhostport: \u003cstring\u003e | default = \"localhost:24225\"] ### `querier_config` # \"ingestermaxquerylookback\" has been renamed to \"query_ingesters_within\". -[ingestermaxquerylookback: \u003cduration\u003e | default = 0s] +[query_ingesters_within: \u003cduration\u003e | default = 0s] ### `queryrange_config` results_cache: cache: # \"defaul_validity\" has been renamed to \"default_validity\". - [defaul_validity: \u003cduration\u003e | default = 0s] + [default_validity: \u003cduration\u003e | default = 0s] # \"cache_split_interval\" has been deprecated in favor of \"split_queries_by_interval\". - [cache_split_interval: \u003cduration\u003e | default = 24h0m0s] ### `alertmanager_config` # The \"store\" config block has been added. This includes \"configdb\" which previously # was the \"configdb\" root level config block. +store: + [type: \u003cstring\u003e | default = \"configdb\"] + [configdb: \u003cconfigstore_config\u003e] + local: + [path: \u003cstring\u003e | default = \"\"] ### `storage_config` index_queries_cache_config: # \"defaul_validity\" has been renamed to \"default_validity\". - [defaul_validity: \u003cduration\u003e | default = 0s] + [default_validity: \u003cduration\u003e | default = 0s] ### `chunk_store_config` chunk_cache_config: # \"defaul_validity\" has been renamed to \"default_validity\". - [defaul_validity: \u003cduration\u003e | default = 0s] + [default_validity: \u003cduration\u003e | default = 0s] write_dedupe_cache_config: # \"defaul_validity\" has been renamed to \"default_validity\". - [defaul_validity: \u003cduration\u003e | default = 0s] + [default_validity: \u003cduration\u003e | default = 0s] # \"min_chunk_age\" has been removed in favor of \"querier \u003e query_store_after\". -[min_chunk_age: \u003cduration\u003e | default = 0s] ### `configs_config` -# \"uri\" has been moved to \"database \u003e uri\". -[uri: \u003cstring\u003e | default = \"postgres://postgres@configs-db.weave.local/configs?sslmode=disable\"] -# \"migrationsdir\" has been moved to \"database \u003e migrations_dir\". -[migrationsdir: \u003cstring\u003e | default = \"\"] -# \"passwordfile\" has been moved to \"database \u003e password_file\". -[passwordfile: \u003cstring\u003e | default = \"\"] +database: + [uri: \u003cstring\u003e | default = \"postgres://postgres@configs-db.weave.local/configs?sslmode=disable\"] + [migrations_dir: \u003cstring\u003e | default = \"\"] + [password_file: \u003cstring\u003e | default = \"\"] 0.6.1 / 2020-02-05 [BUGFIX] Fixed parsing of the WAL configuration when specified in the YAML config file. #2071 0.6.0 / 2020-01-28 Note that the ruler flags need to be changed in this upgrade. You’re moving from a single node ruler to something that might need to be sharded. Further, if you’re using the configs service, we’ve upgraded the migration library and this requires some manual intervention. See full instructions below to upgrade your PostgreSQL.\n[CHANGE] The frontend component now does not cache results if it finds a Cache-Control header and if one of its values is no-store. #1974 [CHANGE] Flags changed with transition to upstream Prometheus rules manager: -ruler.client-timeout is now ruler.configs.client-timeout in order to match ruler.configs.url. -ruler.group-timeouthas been removed. -ruler.num-workers has been removed. -ruler.rule-path has been added to specify where the prometheus rule manager will sync rule files. -ruler.storage.type has beem added to specify the rule store backend type, currently only the configdb. -ruler.poll-interval has been added to specify the interval in which to poll new rule groups. -ruler.evaluation-interval default value has changed from 15s to 1m to match the default evaluation interval in Prometheus. Ruler sharding requires a ring which can be configured via the ring flags prefixed by ruler.ring.. #1987 [CHANGE] Use relative links from /ring page to make it work when used behind reverse proxy. #1896 [CHANGE] Deprecated -distributor.limiter-reload-period flag. #1766 [CHANGE] Ingesters now write only normalised tokens to the ring, although they can still read denormalised tokens used by other ingesters. -ingester.normalise-tokens is now deprecated, and ignored. If you want to switch back to using denormalised tokens, you need to downgrade to Cortex 0.4.0. Previous versions don’t handle claiming tokens from normalised ingesters correctly. #1809 [CHANGE] Overrides mechanism has been renamed to “runtime config”, and is now separate from limits. Runtime config is simply a file that is reloaded by Cortex every couple of seconds. Limits and now also multi KV use this mechanism.\nNew arguments were introduced: -runtime-config.file (defaults to empty) and -runtime-config.reload-period (defaults to 10 seconds), which replace previously used -limits.per-user-override-config and -limits.per-user-override-period options. Old options are still used if -runtime-config.file is not specified. This change is also reflected in YAML configuration, where old limits.per_tenant_override_config and limits.per_tenant_override_period fields are replaced with runtime_config.file and runtime_config.period respectively. #1749 [CHANGE] Cortex now rejects data with duplicate labels. Previously, such data was accepted, with duplicate labels removed with only one value left. #1964 [CHANGE] Changed the default value for -distributor.ha-tracker.prefix from collectors/ to ha-tracker/ in order to not clash with other keys (ie. ring) stored in the same key-value store. #1940 [FEATURE] Experimental: Write-Ahead-Log added in ingesters for more data reliability against ingester crashes. #1103 --ingester.wal-enabled: Setting this to true enables writing to WAL during ingestion. --ingester.wal-dir: Directory where the WAL data should be stored and/or recovered from. --ingester.checkpoint-enabled: Set this to true to enable checkpointing of in-memory chunks to disk. --ingester.checkpoint-duration: This is the interval at which checkpoints should be created. --ingester.recover-from-wal: Set this to true to recover data from an existing WAL. For more information, please checkout the “Ingesters with WAL” guide. [FEATURE] The distributor can now drop labels from samples (similar to the removal of the replica label for HA ingestion) per user via the distributor.drop-label flag. #1726 [FEATURE] Added flag debug.mutex-profile-fraction to enable mutex profiling #1969 [FEATURE] Added global ingestion rate limiter strategy. Deprecated -distributor.limiter-reload-period flag. #1766 [FEATURE] Added support for Microsoft Azure blob storage to be used for storing chunk data. #1913 [FEATURE] Added readiness probe endpoint/ready to queriers. #1934 [FEATURE] Added “multi” KV store that can interact with two other KV stores, primary one for all reads and writes, and secondary one, which only receives writes. Primary/secondary store can be modified in runtime via runtime-config mechanism (previously “overrides”). #1749 [FEATURE] Added support to store ring tokens to a file and read it back on startup, instead of generating/fetching the tokens to/from the ring. This feature can be enabled with the flag -ingester.tokens-file-path. #1750 [FEATURE] Experimental TSDB: Added /series API endpoint support with TSDB blocks storage. #1830 [FEATURE] Experimental TSDB: Added TSDB blocks compactor component, which iterates over users blocks stored in the bucket and compact them according to the configured block ranges. #1942 [ENHANCEMENT] metric cortex_ingester_flush_reasons gets a new reason value: Spread, when -ingester.spread-flushes option is enabled. #1978 [ENHANCEMENT] Added password and enable_tls options to redis cache configuration. Enables usage of Microsoft Azure Cache for Redis service. #1923 [ENHANCEMENT] Upgraded Kubernetes API version for deployments from extensions/v1beta1 to apps/v1. #1941 [ENHANCEMENT] Experimental TSDB: Open existing TSDB on startup to prevent ingester from becoming ready before it can accept writes. The max concurrency is set via --experimental.tsdb.max-tsdb-opening-concurrency-on-startup. #1917 [ENHANCEMENT] Experimental TSDB: Querier now exports aggregate metrics from Thanos bucket store and in memory index cache (many metrics to list, but all have cortex_querier_bucket_store_ or cortex_querier_blocks_index_cache_ prefix). #1996 [ENHANCEMENT] Experimental TSDB: Improved multi-tenant bucket store. #1991 Allowed to configure the blocks sync interval via -experimental.tsdb.bucket-store.sync-interval (0 disables the sync) Limited the number of tenants concurrently synched by -experimental.tsdb.bucket-store.block-sync-concurrency Renamed cortex_querier_sync_seconds metric to cortex_querier_blocks_sync_seconds Track cortex_querier_blocks_sync_seconds metric for the initial sync too [BUGFIX] Fixed unnecessary CAS operations done by the HA tracker when the jitter is enabled. #1861 [BUGFIX] Fixed ingesters getting stuck in a LEAVING state after coming up from an ungraceful exit. #1921 [BUGFIX] Reduce memory usage when ingester Push() errors. #1922 [BUGFIX] Table Manager: Fixed calculation of expected tables and creation of tables from next active schema considering grace period. #1976 [BUGFIX] Experimental TSDB: Fixed ingesters consistency during hand-over when using experimental TSDB blocks storage. #1854 #1818 [BUGFIX] Experimental TSDB: Fixed metrics when using experimental TSDB blocks storage. #1981 #1982 #1990 #1983 [BUGFIX] Experimental memberlist: Use the advertised address when sending packets to other peers of the Gossip memberlist. #1857 [BUGFIX] Experimental TSDB: Fixed incorrect query results introduced in #2604 caused by a buffer incorrectly reused while iterating samples. #2697 Upgrading PostgreSQL (if you’re using configs service) Reference: https://github.com/golang-migrate/migrate/tree/master/database/postgres#upgrading-from-v1\nInstall the migrate package cli tool: https://github.com/golang-migrate/migrate/tree/master/cmd/migrate#installation Drop the schema_migrations table: DROP TABLE schema_migrations;. Run the migrate command: migrate -path \u003cabsolute_path_to_cortex\u003e/cmd/cortex/migrations -database postgres://localhost:5432/database force 2 Known issues The cortex_prometheus_rule_group_last_evaluation_timestamp_seconds metric, tracked by the ruler, is not unregistered for rule groups not being used anymore. This issue will be fixed in the next Cortex release (see 2033).\nWrite-Ahead-Log (WAL) does not have automatic repair of corrupt checkpoint or WAL segments, which is possible if ingester crashes abruptly or the underlying disk corrupts. Currently the only way to resolve this is to manually delete the affected checkpoint and/or WAL segments. Automatic repair will be added in the future releases.\n0.4.0 / 2019-12-02 [CHANGE] The frontend component has been refactored to be easier to reuse. When upgrading the frontend, cache entries will be discarded and re-created with the new protobuf schema. #1734 [CHANGE] Removed direct DB/API access from the ruler. -ruler.configs.url has been now deprecated. #1579 [CHANGE] Removed Delta encoding. Any old chunks with Delta encoding cannot be read anymore. If ingester.chunk-encoding is set to Delta the ingester will fail to start. #1706 [CHANGE] Setting -ingester.max-transfer-retries to 0 now disables hand-over when ingester is shutting down. Previously, zero meant infinite number of attempts. #1771 [CHANGE] dynamo has been removed as a valid storage name to make it consistent for all components. aws and aws-dynamo remain as valid storage names. [CHANGE/FEATURE] The frontend split and cache intervals can now be configured using the respective flag --querier.split-queries-by-interval and --frontend.cache-split-interval. If --querier.split-queries-by-interval is not provided request splitting is disabled by default. --querier.split-queries-by-day is still accepted for backward compatibility but has been deprecated. You should now use --querier.split-queries-by-interval. We recommend a to use a multiple of 24 hours. [FEATURE] Global limit on the max series per user and metric #1760 -ingester.max-global-series-per-user -ingester.max-global-series-per-metric Requires -distributor.replication-factor and -distributor.shard-by-all-labels set for the ingesters too [FEATURE] Flush chunks with stale markers early with ingester.max-stale-chunk-idle. #1759 [FEATURE] EXPERIMENTAL: Added new KV Store backend based on memberlist library. Components can gossip about tokens and ingester states, instead of using Consul or Etcd. #1721 [FEATURE] EXPERIMENTAL: Use TSDB in the ingesters \u0026 flush blocks to S3/GCS ala Thanos. This will let us use an Object Store more efficiently and reduce costs. #1695 [FEATURE] Allow Query Frontend to log slow queries with frontend.log-queries-longer-than. #1744 [FEATURE] Add HTTP handler to trigger ingester flush \u0026 shutdown - used when running as a stateful set with the WAL enabled. #1746 [FEATURE] EXPERIMENTAL: Added GCS support to TSDB blocks storage. #1772 [ENHANCEMENT] Reduce memory allocations in the write path. #1706 [ENHANCEMENT] Consul client now follows recommended practices for blocking queries wrt returned Index value. #1708 [ENHANCEMENT] Consul client can optionally rate-limit itself during Watch (used e.g. by ring watchers) and WatchPrefix (used by HA feature) operations. Rate limiting is disabled by default. New flags added: --consul.watch-rate-limit, and --consul.watch-burst-size. #1708 [ENHANCEMENT] Added jitter to HA deduping heartbeats, configure using distributor.ha-tracker.update-timeout-jitter-max #1534 [ENHANCEMENT] Add ability to flush chunks with stale markers early. #1759 [BUGFIX] Stop reporting successful actions as 500 errors in KV store metrics. #1798 [BUGFIX] Fix bug where duplicate labels can be returned through metadata APIs. #1790 [BUGFIX] Fix reading of old, v3 chunk data. #1779 [BUGFIX] Now support IAM roles in service accounts in AWS EKS. #1803 [BUGFIX] Fixed duplicated series returned when querying both ingesters and store with the experimental TSDB blocks storage. #1778 In this release we updated the following dependencies:\ngRPC v1.25.0 (resulted in a drop of 30% CPU usage when compression is on) jaeger-client v2.20.0 aws-sdk-go to v1.25.22 0.3.0 / 2019-10-11 This release adds support for Redis as an alternative to Memcached, and also includes many optimisations which reduce CPU and memory usage.\n[CHANGE] Gauge metrics were renamed to drop the _total suffix. #1685 In Alertmanager, alertmanager_configs_total is now alertmanager_configs In Ruler, scheduler_configs_total is now scheduler_configs scheduler_groups_total is now scheduler_groups. [CHANGE] --alertmanager.configs.auto-slack-root flag was dropped as auto Slack root is not supported anymore. #1597 [CHANGE] In table-manager, default DynamoDB capacity was reduced from 3,000 units to 1,000 units. We recommend you do not run with the defaults: find out what figures are needed for your environment and set that via -dynamodb.periodic-table.write-throughput and -dynamodb.chunk-table.write-throughput. [FEATURE] Add Redis support for caching #1612 [FEATURE] Allow spreading chunk writes across multiple S3 buckets #1625 [FEATURE] Added /shutdown endpoint for ingester to shutdown all operations of the ingester. #1746 [ENHANCEMENT] Upgraded Prometheus to 2.12.0 and Alertmanager to 0.19.0. #1597 [ENHANCEMENT] Cortex is now built with Go 1.13 #1675, #1676, #1679 [ENHANCEMENT] Many optimisations, mostly impacting ingester and querier: #1574, #1624, #1638, #1644, #1649, #1654, #1702 Full list of changes: https://github.com/cortexproject/cortex/compare/v0.2.0...v0.3.0\n0.2.0 / 2019-09-05 This release has several exciting features, the most notable of them being setting -ingester.spread-flushes to potentially reduce your storage space by upto 50%.\n[CHANGE] Flags changed due to changes upstream in Prometheus Alertmanager #929: alertmanager.mesh.listen-address is now cluster.listen-address alertmanager.mesh.peer.host and alertmanager.mesh.peer.service can be replaced by cluster.peer alertmanager.mesh.hardware-address, alertmanager.mesh.nickname, alertmanager.mesh.password, and alertmanager.mesh.peer.refresh-interval all disappear. [CHANGE] –claim-on-rollout flag deprecated; feature is now always on #1566 [CHANGE] Retention period must now be a multiple of periodic table duration #1564 [CHANGE] The value for the name label for the chunks memcache in all cortex_cache_ metrics is now chunksmemcache (before it was memcache) #1569 [FEATURE] Makes the ingester flush each timeseries at a specific point in the max-chunk-age cycle with -ingester.spread-flushes. This means multiple replicas of a chunk are very likely to contain the same contents which cuts chunk storage space by up to 66%. #1578 [FEATURE] Make minimum number of chunk samples configurable per user #1620 [FEATURE] Honor HTTPS for custom S3 URLs #1603 [FEATURE] You can now point the query-frontend at a normal Prometheus for parallelisation and caching #1441 [FEATURE] You can now specify http_config on alert receivers #929 [FEATURE] Add option to use jump hashing to load balance requests to memcached #1554 [FEATURE] Add status page for HA tracker to distributors #1546 [FEATURE] The distributor ring page is now easier to read with alternate rows grayed out #1621 0.1.0 / 2019-08-07 [CHANGE] HA Tracker flags were renamed to provide more clarity #1465 distributor.accept-ha-labels is now distributor.ha-tracker.enable distributor.accept-ha-samples is now distributor.ha-tracker.enable-for-all-users ha-tracker.replica is now distributor.ha-tracker.replica ha-tracker.cluster is now distributor.ha-tracker.cluster [FEATURE] You can specify “heap ballast” to reduce Go GC Churn #1489 [BUGFIX] HA Tracker no longer always makes a request to Consul/Etcd when a request is not from the active replica #1516 [BUGFIX] Queries are now correctly cancelled by the query-frontend #1508 ","categories":"","description":"","excerpt":"master / unreleased [CHANGE] OTLP: Change OTLP handler to be …","ref":"/docs/changelog/","tags":"","title":"Changelog"},{"body":"Cortex follows the CNCF Code of Conduct.\n","categories":"","description":"","excerpt":"Cortex follows the CNCF Code of Conduct.\n","ref":"/docs/code-of-conduct/","tags":"","title":"Code of Conduct"},{"body":"In this page we’re listing some external resources to learn more about the Cortex blocks storage.\nTalks Scaling Prometheus: How We Got Some Thanos Into Cortex Articles How to switch Cortex from chunks to blocks storage (and why you won’t look back) (Oct 2020) Now GA: Cortex blocks storage for running Prometheus at scale with reduced operational complexity (Oct 2020) Scaling Prometheus: How we’re pushing Cortex blocks storage to its limit and beyond (Aug 2020) How blocks storage in Cortex reduces operational complexity for running Prometheus at massive scale (Jul 2020) ","categories":"","description":"","excerpt":"In this page we’re listing some external resources to learn more about …","ref":"/docs/blocks-storage/learn-more/","tags":"","title":"Learn more"},{"body":"Tenant ID naming The tenant ID (also called “user ID” or “org ID”) is the unique identifier of a tenant within a Cortex cluster. The tenant ID is opaque information to Cortex, which doesn’t make any assumptions on its format/content, but its naming has two limitations:\nSupported characters Length Supported characters The following character sets are generally safe for use in the tenant ID:\nAlphanumeric characters 0-9 a-z A-Z Special characters Exclamation point (!) Hyphen (-) Underscore (_) Single Period (.), but the tenant IDs . and .. are considered invalid Asterisk (*) Single quote (') Open parenthesis (() Close parenthesis ()) All other characters are not safe to use. In particular, slashes / and whitespaces ( ) are not supported.\nLength The tenant ID length should not exceed 150 bytes/characters.\n","categories":"","description":"","excerpt":"Tenant ID naming The tenant ID (also called “user ID” or “org ID”) is …","ref":"/docs/guides/limitations/","tags":"","title":"Limitations"},{"body":"Blocks storage The blocks storage is a Cortex storage engine based on Prometheus TSDB, which only requires an object store (eg. AWS S3, Google GCS, …) as backend storage.\nFor more information, please refer to the Cortex blocks storage documentation.\nChunk A chunk is an object containing compressed timestamp-value pairs.\nA single chunk contains timestamp-value pairs for several series.\nChurn Churn is the frequency at which series become idle.\nA series becomes idle once it’s not exported anymore by the monitored targets. Typically, series become idle when the monitored target itself disappears (eg. the process or node gets terminated).\nFlushing Series flushing is the operation run by ingesters to offload time series from memory and store them in the long-term storage.\nHA Tracker The HA Tracker is a feature of Cortex distributor which is used to deduplicate received series coming from two (or more) Prometheus servers configured in HA pairs.\nFor more information, please refer to the guide “Config for sending HA Pairs data to Cortex”.\nHash ring The hash ring is a distributed data structure used by Cortex for sharding, replication, and service discovery. The hash ring data structure gets shared across Cortex replicas via gossip or a key-value store.\nFor more information, please refer to the Architecture documentation.\nOrg See Tenant.\nRing See Hash ring.\nSample A sample is a single timestamped value in a time series.\nFor example, given the series node_cpu_seconds_total{instance=\"10.0.0.1\",mode=\"system\"} its stream of values (samples) could be:\n# Display format: \u003cvalue\u003e @\u003ctimestamp\u003e 11775 @1603812134 11790 @1603812149 11805 @1603812164 11819 @1603812179 11834 @1603812194 Series In the Prometheus ecosystem, a series (or time series) is a single stream of timestamped values belonging to the same metric, with the same set of label key-value pairs.\nFor example, given a single metric node_cpu_seconds_total you may have multiple series, each one uniquely identified by the combination of metric name and unique label key-value pairs:\nnode_cpu_seconds_total{instance=\"10.0.0.1\",mode=\"system\"} node_cpu_seconds_total{instance=\"10.0.0.1\",mode=\"user\"} node_cpu_seconds_total{instance=\"10.0.0.2\",mode=\"system\"} node_cpu_seconds_total{instance=\"10.0.0.2\",mode=\"user\"} Tenant A tenant (also called “user” or “org”) is the owner of a set of series written to and queried from Cortex. Cortex multi-tenancy support allows you to isolate series belonging to different tenants. For example, if you have two tenants team-A and team-B, team-A series will be isolated from team-B, and each team will be able to query only their own series.\nFor more information, please refer to:\nHTTP API authentication Tenant ID limitations Time series See Series.\nUser See Tenant.\nWAL The Write-Ahead Log (WAL) is an append-only log stored on disk used by ingesters to recover their in-memory state after the process gets restarted, either after a clear shutdown or an abrupt termination.\nFor more information, please refer to Ingesters with WAL.\n","categories":"","description":"","excerpt":"Blocks storage The blocks storage is a Cortex storage engine based on …","ref":"/docs/guides/glossary/","tags":"","title":"Glossary"},{"body":"","categories":"","description":"","excerpt":"","ref":"/index.json","tags":"","title":""},{"body":" Horizontally scalable, highly available, multi-tenant, long term storage for Prometheus. Learn More Releases Companies using Cortex\nLong term storage Durably store data for longer than the lifetime of any single machine, and use this data for long term capacity planning. Blazin’ fast PromQL Cortex makes your PromQL queries blazin' fast through aggressive parallelization and caching. A global view of data Cortex gives you a global view of Prometheus time series data that includes data in long-term storage, greatly expanding the usefulness of PromQL for analytical purposes. Horizontally scalable Cortex runs across multiple machines in a cluster, exceeding the throughput and storage of a single machine. This enables you to send the metrics from multiple Prometheus servers to a single Cortex cluster. We are a Cloud Native Computing Foundation Incubating project.\nJoin the community ! Join users and companies that are using Cortex in production.\nSlack Issues Twitter ","categories":"","description":"","excerpt":" Horizontally scalable, highly available, multi-tenant, long term …","ref":"/","tags":"","title":"Cortex"},{"body":"","categories":"","description":"","excerpt":"","ref":"/search/","tags":"","title":"Search Results"}]
\ No newline at end of file
diff --git a/search/index.html b/search/index.html
index ce2aaea105..9be4cfad90 100644
--- a/search/index.html
+++ b/search/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.101.0"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/favicons/android-96x96.png sizes=96x96><link rel=icon type=image/png href=/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/favicons/android-192x192.png sizes=192x192><title>Search Results | Cortex</title><meta name=description content><meta property="og:title" content="Search Results"><meta property="og:description" content><meta property="og:type" content="article"><meta property="og:url" content="/search/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content><meta property="article:modified_time" content="2024-12-05T13:05:53-08:00"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Search Results"><meta itemprop=description content><meta itemprop=dateModified content="2024-12-05T13:05:53-08:00"><meta itemprop=wordCount content="0"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Search Results"><meta name=twitter:description content><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.101.0"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/favicons/android-96x96.png sizes=96x96><link rel=icon type=image/png href=/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/favicons/android-192x192.png sizes=192x192><title>Search Results | Cortex</title><meta name=description content><meta property="og:title" content="Search Results"><meta property="og:description" content><meta property="og:type" content="article"><meta property="og:url" content="/search/"><meta property="og:image" content="/images/logo-twitter-card.jpg"><meta property="article:section" content><meta property="article:modified_time" content="2024-12-12T06:21:42+09:00"><meta property="og:site_name" content="Cortex"><meta itemprop=name content="Search Results"><meta itemprop=description content><meta itemprop=dateModified content="2024-12-12T06:21:42+09:00"><meta itemprop=wordCount content="0"><meta itemprop=image content="/images/logo-twitter-card.jpg"><meta itemprop=keywords content><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="/images/logo-twitter-card.jpg"><meta name=twitter:title content="Search Results"><meta name=twitter:description content><link rel=preload href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css as=style><link href=/scss/main.min.b2327889d9640582f368772c608732b32a6fa5b2b0c58b3d3c1d139029933f80.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.6.0.min.js integrity=sha384-vtXRMe3mGCbOeY7l30aIg8H9p3GdeSe4IFlP6G8JMa7o7lXvnz3GFKzPxzJdPfGK crossorigin=anonymous></script>
 <script defer src=https://unpkg.com/lunr@2.3.9/lunr.min.js integrity=sha384-203J0SNzyqHby3iU6hzvzltrWi/M41wOP5Gu+BiJMz5nwKykbkUx8Kp7iti0Lpli crossorigin=anonymous></script>
 <script async src="https://www.googletagmanager.com/gtag/js?id=G-SRR0DSREGX"></script>
-<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/docs/><span>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.6ac5dbd40cb5dadece805d646ecf79bf.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-default td-outer"><main role=main class=td-main><section class="row td-search-result"><div class="col-12 col-md-8 offset-md-2"><h2 class=ml-4>Search Results</h2></div></section></main><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
+<script>var doNotTrack=!1;if(!doNotTrack){window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-SRR0DSREGX",{anonymize_ip:!1})}</script></head><body class=td-page><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar navbar-nocover"><a id=cortex-logo class=navbar-brand href=/><span class=navbar-logo><svg xmlns="http://www.w3.org/2000/svg" role="img" viewBox="-17.92 -14.92 1191.84 462.84"><defs><style>.cls-1{fill:#fff}</style></defs><path d="M214.531 8.017C99.353 8.017 5.65 101.72 5.65 216.899S99.353 425.78 214.531 425.78s208.882-93.704 208.882-208.882S329.71 8.017 214.531 8.017zm0 408.558c-110.102.0-199.676-89.574-199.676-199.676S104.429 17.222 214.53 17.222 414.208 106.797 414.208 216.9 324.633 416.575 214.53 416.575z" class="cls-1"/><circle cx="327.452" cy="221.633" r="34.571" class="cls-1"/><circle cx="213.713" cy="353.584" r="34.571" class="cls-1"/><path d="M299.992 167.913l-59-56.05a34.578 34.578.0 10-4.343 4.34l57.828 54.937a60.158 60.158.0 00-27.168 49.123h-12.97l-23.456-67.02-26.055 64.718H175.73l-24.724 57.22-21.808-54.524-.063.025v-.586h-22.048a34.582 34.582.0 10-.275 6.138h18.005l25.97 64.925 28.977-67.06h29.207l21.51-53.424 19.503 55.725H267.5a60.173 60.173.0 0025.202 44.11l-56.52 56.52 4.34 4.338 57.492-57.492a60.155 60.155.0 101.977-105.963zm81.481 53.515a54.02 54.02.0 11-54.022-54.02 54.083 54.083.0 0154.022 54.02zm175.707 42.568q-8.797 8.178-24.405 8.177a34.858 34.858.0 01-17.095-3.965 33.188 33.188.0 01-11.643-10.529 46.374 46.374.0 01-6.566-14.99 71.134 71.134.0 01-2.105-17.341 86.99 86.99.0 011.982-18.706 46.804 46.804.0 016.565-15.98 34.028 34.028.0 0112.263-11.149q7.675-4.21 19.077-4.212 13.38.0 21.306 6.69 7.927 6.689 10.405 18.829h21.803a50.76 50.76.0 00-5.946-19.696 44.06 44.06.0 00-12.015-13.75 49.842 49.842.0 00-16.848-8.051 77.44 77.44.0 00-20.44-2.601q-15.114.0-26.509 5.326a52.935 52.935.0 00-18.952 14.617 62.165 62.165.0 00-11.273 21.8 94.013 94.013.0 00-3.717 26.883 86.195 86.195.0 003.84 26.386 57.783 57.783.0 0011.397 20.686 50.138 50.138.0 0018.829 13.38 66.766 66.766.0 0025.891 4.705q24.527.0 38.772-12.882 14.245-12.878 17.715-36.668h-21.556q-1.986 14.867-10.776 23.041zm157.44-87.828a56.338 56.338.0 00-19.447-14.244q-11.52-5.203-26.882-5.203-15.115.0-26.756 5.203a56.009 56.009.0 00-19.573 14.244 59.75 59.75.0 00-11.892 21.307 85.299 85.299.0 00-3.965 26.386 84.117 84.117.0 003.965 26.262 59.853 59.853.0 0011.892 21.183 54.61 54.61.0 0019.573 14.12q11.643 5.077 26.756 5.08 15.36.0 26.882-5.08a54.908 54.908.0 0019.447-14.12 59.95 59.95.0 0011.892-21.183 84.181 84.181.0 003.965-26.262 85.364 85.364.0 00-3.965-26.386 59.846 59.846.0 00-11.892-21.307zm-9.538 68.38a43.305 43.305.0 01-8.548 15.113 37.061 37.061.0 01-12.759 9.29 38.825 38.825.0 01-30.968.0 37.003 37.003.0 01-12.76-9.29 43.209 43.209.0 01-8.547-15.114 70.669 70.669.0 010-41.373 44.634 44.634.0 018.547-15.237 36.428 36.428.0 0112.76-9.415 38.826 38.826.0 0130.968.0 36.484 36.484.0 0112.76 9.415 44.735 44.735.0 018.547 15.237 70.624 70.624.0 010 41.373zm81.141-80.89q-11.147 7.434-18.83 23.041h-.493v-27.005h-19.82V287.78h21.057v-56.984a87.528 87.528.0 012.478-21.924 41.993 41.993.0 017.93-16.228 33.951 33.951.0 0114.368-10.158q8.919-3.468 21.555-3.468V156.72q-17.097-.493-28.245 6.937zm80.761-42.366h-21.06v38.402h-21.8v18.581h21.8v81.51a48.668 48.668.0 001.734 14.37 17.432 17.432.0 005.326 8.423 20.57 20.57.0 009.415 4.088 75.59 75.59.0 0013.999 1.115h16.104v-18.582h-9.664a70.335 70.335.0 01-8.05-.37 10.361 10.361.0 01-4.833-1.611 6.108 6.108.0 01-2.354-3.469 23.006 23.006.0 01-.617-5.946v-79.528h25.518v-18.581h-25.518zm148.522 60.452a56.135 56.135.0 00-18.085-17.962q-11.276-7.063-28.367-7.06a58.263 58.263.0 00-24.155 4.953A56.796 56.796.0 00925.82 175.55a63.949 63.949.0 00-12.51 21.058 77.064 77.064.0 00-4.461 26.758 102.521 102.521.0 004.338 27.004 58.87 58.87.0 0011.519 21.307 52.43 52.43.0 0018.953 13.873q11.272 4.954 26.632 4.955 21.8.0 36.173-10.901 14.364-10.898 18.58-32.455h-20.81q-2.73 12.635-11.272 18.829-8.549 6.198-21.927 6.195a43.577 43.577.0 01-18.085-3.468 35.417 35.417.0 01-12.636-9.291 36.127 36.127.0 01-7.184-13.38 50.746 50.746.0 01-1.981-15.98h95.879a102.07 102.07.0 00-2.107-24.526 71.064 71.064.0 00-9.415-23.784zm-84.357 29.73a43.81 43.81.0 013.219-13.999 37.354 37.354.0 017.433-11.52 34.013 34.013.0 0111.272-7.803 36.694 36.694.0 0114.74-2.85 36.062 36.062.0 0114.494 2.85 36.492 36.492.0 0111.398 7.68 36.107 36.107.0 017.68 11.52 43.253 43.253.0 013.345 14.122zm170.95 7.184 44.1-58.964h-25.271l-31.961 44.843-30.721-44.843h-27.006l44.595 60.698-48.063 67.389h25.518l35.677-53.019 35.676 53.019h27.006l-49.55-69.123z" class="cls-1"/></svg></span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/docs/><span>Documentation</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://twitter.com/cortexmetrics class="nav-link active"><span class=active><i class="fab fa-fw fa-twitter"></i> Twitter</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a href=https://github.com/cortexproject class="nav-link active"><span class=active><i class="fab fa-fw fa-github"></i> GitHub</span></a></li></ul></div><div class="navbar-nav d-none d-md-block"><input type=search class="form-control td-search-input" placeholder="&#xf002; Search this site…" aria-label="Search this site…" autocomplete=off data-offline-search-index-json-src=/offline-search-index.8cae23c09296d525fd17d4d6f0fcc7af.json data-offline-search-base-href=/ data-offline-search-max-results=10></div></nav></header><div class="container-fluid td-default td-outer"><main role=main class=td-main><section class="row td-search-result"><div class="col-12 col-md-8 offset-md-2"><h2 class=ml-4>Search Results</h2></div></section></main><footer><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5"><div class="col-12 col-sm-4 pb-5 pb-sm-0"><img src=/images/cortex-stacked-white.png width=75px class="img-fluid float-left"></div><div class="col-12 col-sm-4 pb-5 pb-sm-0"><h6 class=font-weight-bold>Community</h6><ul class=list-unstyled><li><a href=https://slack.cncf.io class="text-reset text-white"><i class="fab fa-fw fa-slack"></i> Slack</a></li><li><a href=https://github.com/cortexproject/cortex class="text-reset text-white"><i class="fab fa-fw fa-github"></i> GitHub</a></li><li><a href=https://twitter.com/cortexmetrics class="text-reset text-white"><i class="fab fa-fw fa-twitter"></i> Twitter</a></li></ul></div><div class="col-12 col-sm-4"><h6 class=font-weight-bold>About</h6><p>Cortex is an OSS licensed project as Apache License 2.0</p></div></div><div class="row td-box td-box--dark td-box--gradient td-box--height-auto text-white p-5 text-center"><small class="col-12 text-center">© 2024 The Cortex Authors All Rights Reserved.<br>The Linux Foundation has registered trademarks and uses trademarks.
 For a list of trademarks of The Linux Foundation, please see <a class="text-light alert-link" href=https://www.linuxfoundation.org/trademark-usage>Trademark Usage page</a>.</small></div></footer></div><script src=https://cdn.jsdelivr.net/npm/popper.js@1.16.1/dist/umd/popper.min.js integrity=sha384-9/reFTGAW83EW2RDu2S0VKaIzap3H66lZH81PoYlFhbGU+6BZp6G7niu735Sk7lN crossorigin=anonymous></script>
 <script src=https://cdn.jsdelivr.net/npm/bootstrap@4.6.1/dist/js/bootstrap.min.js integrity="sha512-UR25UO94eTnCVwjbXozyeVd6ZqpaAE9naiEUBK/A+QDbfSTQFhPGj5lOR6d8tsgbBk84Ggb5A3EkjsOgPRPcKA==" crossorigin=anonymous></script>
 <script src=/js/tabpane-persist.js></script>
diff --git a/sitemap.xml b/sitemap.xml
index 295d4ad34b..a53ba97ed8 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -1 +1 @@
-<?xml version="1.0" encoding="utf-8" standalone="yes"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml"><url><loc>/docs/proposals/auth-gateway/</loc></url><url><loc>/docs/proposals/blocks-storage-bucket-index/</loc></url><url><loc>/docs/proposals/blocks-storage-sharding/</loc></url><url><loc>/docs/configuration/configuration-file/</loc></url><url><loc>/docs/proposals/cross-tenant-query-federation/</loc></url><url><loc>/docs/proposals/tenant-deletion/</loc></url><url><loc>/docs/</loc></url><url><loc>/docs/proposals/documentation-versioning/</loc></url><url><loc>/docs/proposals/generalize-modules/</loc></url><url><loc>/docs/getting-started/</loc></url><url><loc>/docs/case-studies/gojek/</loc></url><url><loc>/docs/contributing/governance/</loc></url><url><loc>/docs/proposals/http-api-design/</loc></url><url><loc>/docs/proposals/ingesters-migration/</loc></url><url><loc>/docs/proposals/parallel-compaction/</loc></url><url><loc>/docs/proposals/tenant-retention/</loc></url><url><loc>/docs/proposals/ring-multikey/</loc></url><url><loc>/docs/proposals/ruler-ha/</loc></url><url><loc>/docs/proposals/ruler-high-availability/</loc></url><url><loc>/docs/proposals/ruler-tenant-federation/</loc></url><url><loc>/docs/proposals/scalable-alertmanager/</loc></url><url><loc>/docs/proposals/scalable-query-frontend/</loc></url><url><loc>/docs/proposals/shuffle-sharding-and-zone-awareness/</loc></url><url><loc>/docs/proposals/shuffle-sharding-on-the-read-path/</loc></url><url><loc>/docs/proposals/support-metadata-api/</loc></url><url><loc>/docs/proposals/block-storage-time-series-deletion/</loc></url><url><loc>/docs/proposals/timeseries-partitioning-in-compactor/</loc></url><url><loc>/docs/architecture/</loc></url><url><loc>/docs/configuration/arguments/</loc></url><url><loc>/docs/contributing/design-patterns-and-code-conventions/</loc></url><url><loc>/docs/blocks-storage/querier/</loc></url><url><loc>/docs/operations/query-auditor/</loc></url><url><loc>/docs/case-studies/rewe-digital/</loc></url><url><loc>/docs/blocks-storage/</loc></url><url><loc>/docs/case-studies/buoyant-cloud/</loc></url><url><loc>/docs/contributing/how-to-run-the-website-locally/</loc></url><url><loc>/docs/configuration/prometheus-frontend/</loc></url><url><loc>/docs/operations/query-tee/</loc></url><url><loc>/docs/guides/running-cortex-on-kubernetes/</loc></url><url><loc>/docs/blocks-storage/store-gateway/</loc></url><url><loc>/docs/blocks-storage/compactor/</loc></url><url><loc>/docs/guides/getting-started-with-gossiped-ring/</loc></url><url><loc>/docs/contributing/how-to-upgrade-golang-version/</loc></url><url><loc>/docs/blocks-storage/production-tips/</loc></url><url><loc>/docs/operations/requests-mirroring-to-secondary-cluster/</loc></url><url><loc>/docs/blocks-storage/binary-index-header/</loc></url><url><loc>/docs/blocks-storage/bucket-index/</loc></url><url><loc>/docs/configuration/</loc></url><url><loc>/docs/contributing/how-integration-tests-work/</loc></url><url><loc>/docs/contributing/how-to-update-the-build-image/</loc></url><url><loc>/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/</loc></url><url><loc>/docs/operations/scaling-query-frontend/</loc></url><url><loc>/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/</loc></url><url><loc>/docs/guides/</loc></url><url><loc>/docs/configuration/v1guarantees/</loc></url><url><loc>/docs/api/</loc></url><url><loc>/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/</loc></url><url><loc>/docs/operations/</loc></url><url><loc>/docs/case-studies/</loc></url><url><loc>/docs/proposals/</loc></url><url><loc>/docs/guides/alertmanager-configuration/</loc></url><url><loc>/docs/guides/auth/</loc></url><url><loc>/docs/guides/capacity-planning/</loc></url><url><loc>/docs/guides/ruler-sharding/</loc></url><url><loc>/docs/guides/ha-pair-handling/</loc></url><url><loc>/docs/contributing/</loc></url><url><loc>/docs/guides/encryption-at-rest/</loc></url><url><loc>/docs/contributing/how-to-add-a-maintainer/</loc></url><url><loc>/docs/guides/ingesters-rolling-updates/</loc></url><url><loc>/docs/guides/ingesters-scaling-up-and-down/</loc></url><url><loc>/docs/guides/overrides-exporter/</loc></url><url><loc>/docs/roadmap/</loc></url><url><loc>/docs/guides/tls/</loc></url><url><loc>/docs/guides/security/</loc></url><url><loc>/docs/guides/shuffle-sharding/</loc></url><url><loc>/docs/guides/tracing/</loc></url><url><loc>/docs/guides/zone-aware-replication/</loc></url><url><loc>/docs/changelog/</loc></url><url><loc>/docs/code-of-conduct/</loc></url><url><loc>/docs/blocks-storage/learn-more/</loc></url><url><loc>/docs/guides/limitations/</loc></url><url><loc>/docs/guides/glossary/</loc></url><url><loc>/index.json</loc><lastmod>2024-12-05T13:05:53-08:00</lastmod></url><url><loc>/</loc><lastmod>2024-12-05T13:05:53-08:00</lastmod></url><url><loc>/search/</loc><lastmod>2024-12-05T13:05:53-08:00</lastmod></url></urlset>
\ No newline at end of file
+<?xml version="1.0" encoding="utf-8" standalone="yes"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml"><url><loc>/docs/proposals/auth-gateway/</loc></url><url><loc>/docs/proposals/blocks-storage-bucket-index/</loc></url><url><loc>/docs/proposals/blocks-storage-sharding/</loc></url><url><loc>/docs/configuration/configuration-file/</loc></url><url><loc>/docs/proposals/cross-tenant-query-federation/</loc></url><url><loc>/docs/proposals/tenant-deletion/</loc></url><url><loc>/docs/</loc></url><url><loc>/docs/proposals/documentation-versioning/</loc></url><url><loc>/docs/proposals/generalize-modules/</loc></url><url><loc>/docs/getting-started/</loc></url><url><loc>/docs/case-studies/gojek/</loc></url><url><loc>/docs/contributing/governance/</loc></url><url><loc>/docs/proposals/http-api-design/</loc></url><url><loc>/docs/proposals/ingesters-migration/</loc></url><url><loc>/docs/proposals/parallel-compaction/</loc></url><url><loc>/docs/proposals/tenant-retention/</loc></url><url><loc>/docs/proposals/ring-multikey/</loc></url><url><loc>/docs/proposals/ruler-ha/</loc></url><url><loc>/docs/proposals/ruler-high-availability/</loc></url><url><loc>/docs/proposals/ruler-tenant-federation/</loc></url><url><loc>/docs/proposals/scalable-alertmanager/</loc></url><url><loc>/docs/proposals/scalable-query-frontend/</loc></url><url><loc>/docs/proposals/shuffle-sharding-and-zone-awareness/</loc></url><url><loc>/docs/proposals/shuffle-sharding-on-the-read-path/</loc></url><url><loc>/docs/proposals/support-metadata-api/</loc></url><url><loc>/docs/proposals/block-storage-time-series-deletion/</loc></url><url><loc>/docs/proposals/timeseries-partitioning-in-compactor/</loc></url><url><loc>/docs/architecture/</loc></url><url><loc>/docs/configuration/arguments/</loc></url><url><loc>/docs/contributing/design-patterns-and-code-conventions/</loc></url><url><loc>/docs/blocks-storage/querier/</loc></url><url><loc>/docs/operations/query-auditor/</loc></url><url><loc>/docs/case-studies/rewe-digital/</loc></url><url><loc>/docs/blocks-storage/</loc></url><url><loc>/docs/case-studies/buoyant-cloud/</loc></url><url><loc>/docs/contributing/how-to-run-the-website-locally/</loc></url><url><loc>/docs/configuration/prometheus-frontend/</loc></url><url><loc>/docs/operations/query-tee/</loc></url><url><loc>/docs/guides/running-cortex-on-kubernetes/</loc></url><url><loc>/docs/blocks-storage/store-gateway/</loc></url><url><loc>/docs/blocks-storage/compactor/</loc></url><url><loc>/docs/guides/getting-started-with-gossiped-ring/</loc></url><url><loc>/docs/contributing/how-to-upgrade-golang-version/</loc></url><url><loc>/docs/blocks-storage/production-tips/</loc></url><url><loc>/docs/operations/requests-mirroring-to-secondary-cluster/</loc></url><url><loc>/docs/blocks-storage/binary-index-header/</loc></url><url><loc>/docs/blocks-storage/bucket-index/</loc></url><url><loc>/docs/configuration/</loc></url><url><loc>/docs/contributing/how-integration-tests-work/</loc></url><url><loc>/docs/contributing/how-to-update-the-build-image/</loc></url><url><loc>/docs/blocks-storage/migrate-cortex-cluster-from-chunks-to-blocks/</loc></url><url><loc>/docs/operations/scaling-query-frontend/</loc></url><url><loc>/docs/blocks-storage/convert-long-term-storage-from-chunks-to-blocks/</loc></url><url><loc>/docs/guides/</loc></url><url><loc>/docs/configuration/v1guarantees/</loc></url><url><loc>/docs/api/</loc></url><url><loc>/docs/blocks-storage/migrate-storage-from-thanos-and-prometheus/</loc></url><url><loc>/docs/operations/</loc></url><url><loc>/docs/case-studies/</loc></url><url><loc>/docs/proposals/</loc></url><url><loc>/docs/guides/alertmanager-configuration/</loc></url><url><loc>/docs/guides/auth/</loc></url><url><loc>/docs/guides/capacity-planning/</loc></url><url><loc>/docs/guides/ruler-sharding/</loc></url><url><loc>/docs/guides/ha-pair-handling/</loc></url><url><loc>/docs/contributing/</loc></url><url><loc>/docs/guides/encryption-at-rest/</loc></url><url><loc>/docs/contributing/how-to-add-a-maintainer/</loc></url><url><loc>/docs/guides/ingesters-rolling-updates/</loc></url><url><loc>/docs/guides/ingesters-scaling-up-and-down/</loc></url><url><loc>/docs/guides/overrides-exporter/</loc></url><url><loc>/docs/roadmap/</loc></url><url><loc>/docs/guides/tls/</loc></url><url><loc>/docs/guides/security/</loc></url><url><loc>/docs/guides/shuffle-sharding/</loc></url><url><loc>/docs/guides/tracing/</loc></url><url><loc>/docs/guides/zone-aware-replication/</loc></url><url><loc>/docs/changelog/</loc></url><url><loc>/docs/code-of-conduct/</loc></url><url><loc>/docs/blocks-storage/learn-more/</loc></url><url><loc>/docs/guides/limitations/</loc></url><url><loc>/docs/guides/glossary/</loc></url><url><loc>/index.json</loc><lastmod>2024-12-12T06:21:42+09:00</lastmod></url><url><loc>/</loc><lastmod>2024-12-12T06:21:42+09:00</lastmod></url><url><loc>/search/</loc><lastmod>2024-12-12T06:21:42+09:00</lastmod></url></urlset>
\ No newline at end of file
