diff --git a/Makefile b/Makefile
index 9cad6c495..37086c989 100644
--- a/Makefile
+++ b/Makefile
@@ -90,7 +90,7 @@ clean:
 
 .PHONY: test
 test: $(PROMTOOL_BIN) prometheus_alerts.yaml prometheus_rules.yaml
-	@$(PROMTOOL_BIN) test rules tests.yaml
+	@$(PROMTOOL_BIN) test rules tests/*.yaml
 
 $(BIN_DIR):
 	mkdir -p $(BIN_DIR)
diff --git a/tests/apiserver-availability-test.yaml b/tests/apiserver-availability-test.yaml
new file mode 100644
index 000000000..6bb36267c
--- /dev/null
+++ b/tests/apiserver-availability-test.yaml
@@ -0,0 +1,23 @@
+rule_files:
+- ../prometheus_alerts.yaml
+- ../prometheus_rules.yaml
+
+evaluation_interval: 1m
+
+tests:
+- interval: 1m
+  input_series:
+  # 100 requests in the overall interval, 99 successful and 1 error
+  - series: 'apiserver_request_total{job="kube-apiserver",verb="GET",code="200"}'
+    values: '0 10 20 50 90 99'
+  - series: 'apiserver_request_total{job="kube-apiserver",verb="GET",code="500"}'
+    values: '0 0 0 1 1 1'
+
+  promql_expr_test:
+  - eval_time: 5m
+    expr: code_verb:apiserver_request_total:increase1h{verb="GET"}
+    exp_samples:
+    - labels: '{__name__="code_verb:apiserver_request_total:increase1h", code="200", verb="GET"}'
+      value: 99.0
+    - labels: '{__name__="code_verb:apiserver_request_total:increase1h", code="500", verb="GET"}'
+      value: 1.0
diff --git a/tests.yaml b/tests/tests.yaml
similarity index 99%
rename from tests.yaml
rename to tests/tests.yaml
index 44d422ff5..66acd7719 100644
--- a/tests.yaml
+++ b/tests/tests.yaml
@@ -1,6 +1,6 @@
 rule_files:
-  - prometheus_alerts.yaml
-  - prometheus_rules.yaml
+- ../prometheus_alerts.yaml
+- ../prometheus_rules.yaml
 
 evaluation_interval: 1m