Merge branch 'master' into mayday

.cspell.json

@@ -66,9 +66,27 @@
         "earlyoom",
         "nvme",
         "nofail",
+        "cloudimg",
+        "fstype",
+        "imds",
+        "mysqld",
+        "nvme",
+        "ocpus",
+        "rootfs",
+        "uefi",
+        "vcpu",
+        "vcpus",
+        "vfat",
+        "vnic",
+        "vpus",
+        "webp",
         "hset",
         "serializability",
-        "oom"
+        "oom",
+        "jemalloc",
+        "nofail",
+        "earlyoom",
+        "sysrq"
     ],
     "allowCompoundWords": true,
     "ignorePaths": [

.pre-commit-config.yaml

@@ -57,7 +57,7 @@ repos:
 
       - id: cspell-modified-files
         name: check spelling of files
-        entry: sh -c "npx cspell --config .cspell.json `git diff --cached -p --name-status | cut -c3- | tr '\n' ' '`"
+        entry: sh -c "npx cspell --no-must-find-files --config .cspell.json `git diff --cached -p --name-status | cut -c3- | tr '\n' ' '`"
         language: system
         stages: [pre-commit]
 

dashboard/src2/components/server/ServerCharts.vue

@@ -98,7 +98,7 @@
 					:key="spaceData"
 					:data="spaceData"
 					unit="%"
-					:chartTheme="[$theme.colors.red[500]]"
+					:chartTheme="[$theme.colors.red[500], $theme.colors.yellow[400]]"
 					:loading="$resources.space.loading"
 					:error="$resources.space.error"
 					:showCard="false"
@@ -128,7 +128,7 @@
 					:key="iopsData"
 					:data="iopsData"
 					unit="I0ps"
-					:chartTheme="[$theme.colors.purple[500]]"
+					:chartTheme="[$theme.colors.purple[500], $theme.colors.blue[500]]"
 					:loading="$resources.iops.loading"
 					:error="$resources.iops.error"
 					:showCard="false"
@@ -718,13 +718,13 @@ export default {
 			let iops = this.$resources.iops.data;
 			if (!iops) return;
 
-			return this.transformSingleLineChartData(iops);
+			return this.transformMultiLineChartData(iops, null, true);
 		},
 		spaceData() {
 			let space = this.$resources.space.data;
 			if (!space) return;
 
-			return this.transformSingleLineChartData(space, true);
+			return this.transformMultiLineChartData(space, null, true);
 		},
 		networkData() {
 			let network = this.$resources.network.data;

press/api/server.py

@@ -29,6 +29,9 @@ def poly_get_doc(doctypes, name):
 	return frappe.get_doc(doctypes[-1], name)
 
 
+MOUNTPOINT_REGEX = "(/|/opt/volumes/mariadb|/opt/volumes/benches)"
+
+
 @frappe.whitelist()
 def all(server_filter=None):  # noqa: C901
 	if server_filter is None:
@@ -193,7 +196,7 @@ def usage(name):
 			lambda x: x,
 		),
 		"disk": (
-			f"""(node_filesystem_size_bytes{{instance="{name}", job="node", mountpoint="/"}} - node_filesystem_avail_bytes{{instance="{name}", job="node", mountpoint="/"}}) / (1024 * 1024 * 1024)""",
+			f"""sum(node_filesystem_size_bytes{{instance="{name}", job="node", mountpoint=~"{MOUNTPOINT_REGEX}"}} - node_filesystem_avail_bytes{{instance="{name}", job="node", mountpoint=~"{MOUNTPOINT_REGEX}"}}) by ()/ (1024 * 1024 * 1024)""",
 			lambda x: x,
 		),
 		"memory": (
@@ -218,7 +221,7 @@ def total_resource(name):
 			lambda x: x,
 		),
 		"disk": (
-			f"""(node_filesystem_size_bytes{{instance="{name}", job="node", mountpoint="/"}}) / (1024 * 1024 * 1024)""",
+			f"""sum(node_filesystem_size_bytes{{instance="{name}", job="node", mountpoint=~"{MOUNTPOINT_REGEX}"}}) by () / (1024 * 1024 * 1024)""",
 			lambda x: x,
 		),
 		"memory": (
@@ -289,7 +292,7 @@ def analytics(name, query, timezone, duration):
 			lambda x: x["device"],
 		),
 		"space": (
-			f"""100 - ((node_filesystem_avail_bytes{{instance="{name}", job="node", mountpoint="/"}} * 100) / node_filesystem_size_bytes{{instance="{name}", job="node", mountpoint="/"}})""",
+			f"""100 - ((node_filesystem_avail_bytes{{instance="{name}", job="node", mountpoint=~"{MOUNTPOINT_REGEX}"}} * 100) / node_filesystem_size_bytes{{instance="{name}", job="node", mountpoint=~"{MOUNTPOINT_REGEX}"}})""",
 			lambda x: x["mountpoint"],
 		),
 		"loadavg": (
@@ -549,7 +552,7 @@ def rename(name, title):
 	doc.save()
 
 
-def get_timespan_timegrain(duration: str) -> Tuple[int, int]:
+def get_timespan_timegrain(duration: str) -> tuple[int, int]:
 	timespan, timegrain = {
 		"1 Hour": (60 * 60, 2 * 60),
 		"6 Hour": (6 * 60 * 60, 5 * 60),

press/fixtures/press_job_type.json

@@ -2,7 +2,7 @@
  {
   "docstatus": 0,
   "doctype": "Press Job Type",
-  "modified": "2024-07-30 16:26:28.217044",
+  "modified": "2025-01-02 15:31:03.095292",
   "name": "Create Server",
   "steps": [
    {
@@ -61,7 +61,7 @@
     "wait_until_true": 1
    },
    {
-    "script": "if doc.server_type == \"Database Server\":\n    default_variables = frappe.get_all(\"MariaDB Variable\", {\"set_on_new_servers\":1}, pluck=\"name\")\n    for var_name in default_variables:\n        var = frappe.get_doc(\"MariaDB Variable\", var_name)\n        var.set_on_server(doc.server)\n\nserver = frappe.get_doc(doc.server_type, doc.server)\nserver.set_swappiness()\nserver.add_glass_file()\nserver.update_filebeat()\n\nif doc.server_type == \"Server\":\n    server.setup_mysqldump()\n    server.install_earlyoom()\n",
+    "script": "if doc.server_type == \"Database Server\":\n    default_variables = frappe.get_all(\"MariaDB Variable\", {\"set_on_new_servers\":1}, pluck=\"name\")\n    for var_name in default_variables:\n        var = frappe.get_doc(\"MariaDB Variable\", var_name)\n        var.set_on_server(doc.server)\n\nserver = frappe.get_doc(doc.server_type, doc.server)\nserver.set_swappiness()\nserver.add_glass_file()\nserver.update_filebeat()\n\nif doc.server_type == \"Server\":\n    server.setup_mysqldump()\n    server.install_earlyoom()\n\nserver.validate_mounts()\nserver.save()",
     "step_name": "Set additional config",
     "wait_until_true": 0
    }
@@ -70,7 +70,7 @@
  {
   "docstatus": 0,
   "doctype": "Press Job Type",
-  "modified": "2024-12-06 10:58:10.573186",
+  "modified": "2025-01-02 15:33:27.252601",
   "name": "Resize Server",
   "steps": [
    {
@@ -109,7 +109,7 @@
     "wait_until_true": 0
    },
    {
-    "script": "machine = frappe.get_doc(\"Virtual Machine\", doc.virtual_machine)\n\nserver = frappe.get_doc(doc.server_type, doc.server)\nif server.plan:\n    plan_disk_size = frappe.db.get_value(\"Plan\", server.plan, \"disk\")\n    if plan_disk_size and plan_disk_size > machine.disk_size:\n        try:\n            server.increase_disk_size(increment=plan_disk_size - machine.disk_size)\n        except:\n            pass",
+    "script": "machine = frappe.get_doc(\"Virtual Machine\", doc.virtual_machine)\n\nserver = frappe.get_doc(doc.server_type, doc.server)\nif server.plan:\n    plan_disk_size = frappe.db.get_value(\"Server Plan\", server.plan, \"disk\")\n    if plan_disk_size and plan_disk_size > machine.disk_size:\n        try:\n            server.increase_disk_size(increment=plan_disk_size - machine.disk_size)\n        except:\n            pass",
     "step_name": "Increase Disk Size",
     "wait_until_true": 0
    }
@@ -212,11 +212,11 @@
  {
   "docstatus": 0,
   "doctype": "Press Job Type",
-  "modified": "2024-08-07 10:54:29.752755",
+  "modified": "2024-12-26 15:15:23.765651",
   "name": "Increase Disk Size",
   "steps": [
    {
-    "script": "server = frappe.get_doc(doc.server_type, doc.server)\nserver.calculated_increase_disk_size()\n    ",
+    "script": "server = frappe.get_doc(doc.server_type, doc.server)\nmountpoint = arguments.labels.get(\"mountpoint\")\nserver.calculated_increase_disk_size(mountpoint=mountpoint)",
     "step_name": "Increase Disk Size",
     "wait_until_true": 0
    },

press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.py

@@ -384,8 +384,10 @@ def reset_virtual_machine_attributes(self) -> StepStatus:
 
 		# Set new machine image and machine type
 		machine.virtual_machine_image = self.virtual_machine_image
+		machine.machine_image = None
 		machine.machine_type = self.machine_type
-		machine.disk_size = 10  # Default disk size for new machines
+		machine.root_disk_size = 10  # Default root disk size for new machines
+		machine.has_data_volume = True  # VM Migration always adds a data volume
 		machine.save()
 		return StepStatus.Success
 

press/patches.txt

@@ -135,3 +135,5 @@ press.press.doctype.server.patches.set_plan_and_subscription
 press.patches.v0_7_0.move_site_db_access_users_to_site_db_perm_manager
 press.press.doctype.drip_email.patches.set_correct_field_for_html
 press.patches.v0_7_0.set_label_for_site_database_user
+press.press.doctype.virtual_machine.patches.set_root_disk_size
+press.press.doctype.virtual_machine_image.patches.set_root_size

press/playbooks/roles/extend_ec2_volume/tasks/main.yml

@@ -5,16 +5,30 @@
 - name: Show Block Devices
   command: lsblk
 
+- name: Show Partitions
+  command: 'lsblk --noheadings --output PTTYPE {{ device }}'
+  register: lsblk_output
+
+- name: Set Partition Status
+  set_fact:
+    partitioned_disk: '{{ lsblk_output.stdout.strip() != "" }}'
+
 - name: Extend Partition
-  command: growpart /dev/nvme0n1 1
+  command: 'growpart {{ device }} 1'
   retries: 10
   delay: 10
+  when: partitioned_disk
 
 - name: Show Modified Block Devices
   command: lsblk
 
-- name: Extend Filesystem
-  command: resize2fs /dev/nvme0n1p1
+- name: Extend Partitioned Filesystem
+  command: 'resize2fs {{ device }}-part1'
+  when: partitioned_disk
+
+- name: Extend Un-partitioned Filesystem
+  command: 'resize2fs {{ device }}'
+  when: not partitioned_disk
 
 - name: Add Glass file back
   command: fallocate -l 250M /root/glass

press/press/doctype/alertmanager_webhook_log/alertmanager_webhook_log.py

@@ -7,6 +7,7 @@
 from typing import TYPE_CHECKING
 
 import frappe
+from frappe.core.utils import find
 from frappe.model.document import Document
 from frappe.utils import get_url_to_form
 from frappe.utils.background_jobs import enqueue_doc
@@ -18,7 +19,6 @@
 from press.utils import log_error
 
 if TYPE_CHECKING:
-	from press.press.doctype.press_job.press_job import PressJob
 	from press.press.doctype.prometheus_alert_rule.prometheus_alert_rule import (
 		PrometheusAlertRule,
 	)
@@ -123,13 +123,15 @@ def after_insert(self):
 				deduplicate=True,
 			)
 
-	def react_for_instance(self, instance) -> "PressJob":
+	def react_for_instance(self, instance) -> dict:
 		instance_type = self.guess_doctype(instance)
 		if not instance_type:
 			# Prometheus is monitoring instances we don't know about
-			return
+			return {}
 		rule: "PrometheusAlertRule" = frappe.get_doc("Prometheus Alert Rule", self.alert)
-		rule.react(instance_type, instance)
+		labels = self.get_labels_for_instance(instance)
+		job = rule.react(instance_type, instance, labels)
+		return {"press_job_type": job.job_type, "press_job": job.name}
 
 	def react(self):
 		for instance in self.get_instances_from_alerts_payload(self.payload):
@@ -142,6 +144,14 @@ def get_instances_from_alerts_payload(self, payload: str) -> set[str]:
 		instances.extend([alert["labels"]["instance"] for alert in payload["alerts"]])  # sites
 		return set(instances)
 
+	def get_labels_for_instance(self, instance: str) -> dict:
+		# Find first alert that matches the instance
+		payload = json.loads(self.payload)
+		alert = find(payload["alerts"], lambda x: x["labels"]["instance"] == instance)
+		if alert:
+			return alert["labels"]
+		return {}
+
 	def get_past_alert_instances(self):
 		past_alerts = frappe.get_all(
 			self.doctype,

press/press/doctype/alertmanager_webhook_log_reaction_job/alertmanager_webhook_log_reaction_job.json

@@ -12,20 +12,24 @@
   {
    "fieldname": "press_job_type",
    "fieldtype": "Link",
+   "in_list_view": 1,
    "label": "Press Job Type",
-   "options": "Press Job Type"
+   "options": "Press Job Type",
+   "read_only": 1
   },
   {
    "fieldname": "press_job",
    "fieldtype": "Link",
+   "in_list_view": 1,
    "label": "Press Job",
-   "options": "Press Job"
+   "options": "Press Job",
+   "read_only": 1
   }
  ],
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2024-04-30 15:38:54.334429",
+ "modified": "2024-12-26 13:54:57.631772",
  "modified_by": "Administrator",
  "module": "Press",
  "name": "Alertmanager Webhook Log Reaction Job",

press/press/doctype/database_server/database_server.json

@@ -55,6 +55,7 @@
   "tags_section",
   "tags",
   "mounts_section",
+  "has_data_volume",
   "mounts",
   "mariadb_settings_tab",
   "memory_limits_section",
@@ -530,11 +531,19 @@
    "fieldtype": "Table",
    "label": "Mounts",
    "options": "Server Mount"
+  },
+  {
+   "default": "0",
+   "fetch_from": "virtual_machine.has_data_volume",
+   "fieldname": "has_data_volume",
+   "fieldtype": "Check",
+   "label": "Has Data Volume",
+   "read_only": 1
   }
  ],
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2024-12-09 19:35:23.683437",
+ "modified": "2025-01-02 16:51:08.113931",
  "modified_by": "Administrator",
  "module": "Press",
  "name": "Database Server",

press/press/doctype/database_server/database_server.py

@@ -40,6 +40,7 @@ class DatabaseServer(BaseServer):
 		domain: DF.Link | None
 		frappe_public_key: DF.Code | None
 		frappe_user_password: DF.Password | None
+		has_data_volume: DF.Check
 		hostname: DF.Data
 		hostname_abbreviation: DF.Data | None
 		ip: DF.Data | None
@@ -414,7 +415,7 @@ def _get_config(self):
 
 	@frappe.whitelist()
 	def setup_essentials(self):
-		"""Setup missing esessiong after server setup"""
+		"""Setup missing essentials after server setup"""
 		config = self._get_config()
 
 		try:
@@ -449,13 +450,13 @@ def setup_essentials(self):
 
 	def process_hybrid_server_setup(self):
 		try:
-			hybird_server = frappe.db.get_value("Self Hosted Server", {"database_server": self.name}, "name")
+			hybrid_server = frappe.db.get_value("Self Hosted Server", {"database_server": self.name}, "name")
 
-			if hybird_server:
-				hybird_server = frappe.get_doc("Self Hosted Server", hybird_server)
+			if hybrid_server:
+				hybrid_server = frappe.get_doc("Self Hosted Server", hybrid_server)
 
-				if not hybird_server.different_database_server:
-					hybird_server._setup_app_server()
+				if not hybrid_server.different_database_server:
+					hybrid_server._setup_app_server()
 		except Exception:
 			log_error("Hybrid Server Setup exception", server=self.as_dict())
 

press/press/doctype/prometheus_alert_rule/prometheus_alert_rule.py

@@ -114,10 +114,12 @@ def get_routes(self):
 
 		return routes_dict
 
-	def react(self, instance_type: str, instance: str):
-		return self.run_press_job(self.press_job_type, instance_type, instance)
+	def react(self, instance_type: str, instance: str, labels: dict | None = None):
+		return self.run_press_job(self.press_job_type, instance_type, instance, labels)
 
-	def run_press_job(self, job_name: str, server_type: str, server_name: str, arguments=None):
+	def run_press_job(
+		self, job_name: str, server_type: str, server_name: str, labels: dict | None = None, arguments=None
+	):
 		server: "Server" = frappe.get_doc(server_type, server_name)
 		if self.only_on_shared and not server.public:
 			return None
@@ -127,6 +129,11 @@ def run_press_job(self, job_name: str, server_type: str, server_name: str, argum
 		if arguments is None:
 			arguments = {}
 
+		if not labels:
+			labels = {}
+
+		arguments.update({"labels": labels})
+
 		if existing_jobs := frappe.get_all(
 			"Press Job",
 			{