base telemetry

.github/workflows/backend_check.yml

@@ -27,7 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Rust stable toolchain
         uses: actions-rs/toolchain@v1
@@ -37,7 +37,7 @@ jobs:
           override: true
 
       - name: Rust Cache
-        uses: Swatinem/rust-cache@2656b87321093db1cb55fbd73183d195214fdfd1 # v2.5.0
+        uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1
         with:
           workspaces: backend
 

.github/workflows/backend_docs.yml

@@ -27,7 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Rust stable toolchain
         uses: actions-rs/toolchain@v1
@@ -37,7 +37,7 @@ jobs:
           override: true
 
       - name: Rust Cache
-        uses: Swatinem/rust-cache@2656b87321093db1cb55fbd73183d195214fdfd1 # v2.5.0
+        uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1
         with:
           workspaces: backend
 

.github/workflows/backend_fmt.yml

@@ -27,7 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Rust stable toolchain
         uses: actions-rs/toolchain@v1

.github/workflows/backend_tests.yml

@@ -27,7 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Rust stable toolchain
         uses: actions-rs/toolchain@v1
@@ -37,7 +37,7 @@ jobs:
           override: true
 
       - name: Rust Cache
-        uses: Swatinem/rust-cache@2656b87321093db1cb55fbd73183d195214fdfd1 # v2.5.0
+        uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1
         with:
           workspaces: backend
 

.github/workflows/frontend.yml

@@ -28,10 +28,10 @@ jobs:
         node-version: [14.x]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
 

.github/workflows/gitspiegel-trigger.yml

@@ -0,0 +1,22 @@
+name: gitspiegel sync
+
+# This workflow doesn't do anything, it's only use is to trigger "workflow_run"
+# webhook, that'll be consumed by gitspiegel
+# This way, gitspiegel won't do mirroring, unless this workflow runs,
+# and running the workflow is protected by GitHub
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - unlocked
+      - ready_for_review
+      - reopened
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Do nothing
+        run: echo "let's go"

.gitlab-ci.yml

@@ -74,8 +74,8 @@ stages:
         --format=docker \
         --tag "$CONTAINER_REPO:$DOCKER_IMAGE_TAG" "$DOCKERFILE_DIRECTORY"
       fi
-    - echo ${Docker_Hub_Pass_Parity} |
-        buildah login --username ${Docker_Hub_User_Parity} --password-stdin docker.io
+    - echo ${DOCKER_PASSWORD} |
+        buildah login --username ${DOCKER_USERNAME} --password-stdin docker.io
     - |-
       echo pushing "$CONTAINER_REPO:$DOCKER_IMAGE_TAG"
       if [[ $BUILD_LATEST_IMAGE ]]; then
@@ -154,12 +154,16 @@ build-backend:
   variables:
     CONTAINER_REPO:       "docker.io/parity/substrate-telemetry-backend"
     DOCKERFILE_DIRECTORY: "./backend/"
+    DOCKER_USERNAME:      $Docker_Hub_User_Parity
+    DOCKER_PASSWORD:      $Docker_Hub_Pass_Parity
   <<:                     *dockerize
 
 build-frontend:
   variables:
     CONTAINER_REPO:       "docker.io/parity/substrate-telemetry-frontend"
     DOCKERFILE_DIRECTORY: "./frontend/"
+    DOCKER_USERNAME:      $Docker_Hub_User_Parity
+    DOCKER_PASSWORD:      $Docker_Hub_Pass_Parity
   <<:                     *dockerize
 
 publish-backend-docker-image-description:
@@ -175,17 +179,41 @@ publish-frontend-docker-image-description:
     SHORT_DESCRIPTION:    "substrate-frontend Docker Image."
 
 # Manually build the docker images and deploy some commit to staging.
+build-backend-manual:
+  variables:
+    CONTAINER_REPO:       "docker.io/paritypr/substrate-telemetry-backend"
+    DOCKERFILE_DIRECTORY: "./backend/"
+    DOCKER_USERNAME:      $PARITYPR_USER
+    DOCKER_PASSWORD:      $PARITYPR_PASS
+  <<:                     *dockerize
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /^[0-9]+/'
+      when: manual
+
+build-frontend-manual:
+  variables:
+    CONTAINER_REPO:       "docker.io/paritypr/substrate-telemetry-frontend"
+    DOCKERFILE_DIRECTORY: "./frontend/"
+    DOCKER_USERNAME:      $PARITYPR_USER
+    DOCKER_PASSWORD:      $PARITYPR_PASS
+  <<:                     *dockerize
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /^[0-9]+/'
+      when: manual
+
 deploy-commit-to-staging:
   variables:
-    CONTAINER_REPO_BACKEND:  "docker.io/parity/substrate-telemetry-backend"
-    CONTAINER_REPO_FRONTEND: "docker.io/parity/substrate-telemetry-frontend"
+    CONTAINER_REPO_BACKEND:  "docker.io/paritypr/substrate-telemetry-backend"
+    CONTAINER_REPO_FRONTEND: "docker.io/paritypr/substrate-telemetry-frontend"
   stage:                  deploy-commit-to-staging
   <<:                     *deploy
   environment:
     name:                 parity-stg
-  when: manual
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /^[0-9]+/'
+      when: manual
 
-# Autoamtically deploy `master` branch or tag like `v1.0` to staging
+# Automatically deploy `master` branch or tag like `v1.0` to staging
 deploy-master-to-staging:
   variables:
     CONTAINER_REPO_BACKEND:  "docker.io/parity/substrate-telemetry-backend"

backend/telemetry_core/src/feed_message.rs

@@ -240,4 +240,5 @@ pub struct ChainStats {
     pub memory_memcpy_score: Ranking<(u32, Option<u32>)>,
     pub disk_sequential_write_score: Ranking<(u32, Option<u32>)>,
     pub disk_random_write_score: Ranking<(u32, Option<u32>)>,
+    pub cpu_vendor: Ranking<String>,
 }

backend/telemetry_core/src/find_location.rs

@@ -81,7 +81,13 @@ struct Locator {
 }
 
 impl Locator {
-    /// Taken from here: https://github.com/P3TERX/GeoLite.mmdb/releases/tag/2022.06.07
+    /// GeoLite database release data: 2023-10-13
+    ///
+    /// To download the latest version visit: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data.
+    ///
+    /// This database incorporates GeoNames [https://www.geonames.org] geographical data, which is made available
+    /// under the Creative Commons Attribution 4.0 License.
+    /// To view a copy of this license,visit https://creativecommons.org/licenses/by/4.0/.
     const CITY_DATA: &'static [u8] = include_bytes!("GeoLite2-City.mmdb");
 
     pub fn new(cache: FxHashMap<IpAddr, Arc<NodeLocation>>) -> Self {
@@ -138,6 +144,6 @@ mod tests {
     fn locate_random_ip() {
         let ip = "12.5.56.25".parse().unwrap();
         let node_location = Locator::new(Default::default()).locate(ip).unwrap();
-        assert_eq!(&*node_location.city, "El Paso");
+        assert_eq!(&*node_location.city, "Riverside");
     }
 }

backend/telemetry_core/src/state/chain_stats.rs

@@ -96,6 +96,41 @@ fn bucket_memory(memory: u64) -> (u32, Option<u32>) {
         48,
         56,
         64,
+        128,
+    }
+}
+
+fn kernel_version_number(version: &Box<str>) -> &str {
+    let index = version
+        .find("-")
+        .or_else(|| version.find("+"))
+        .unwrap_or(version.len());
+
+    &version[0..index]
+}
+
+#[test]
+fn test_kernel_version_number() {
+    assert_eq!(kernel_version_number(&"5.10.0-8-amd64".into()), "5.10.0");
+    // Plus sign indicates that the kernel was built from modified sources.
+    // This should only appear at the end of the version string.
+    assert_eq!(kernel_version_number(&"5.10.0+82453".into()), "5.10.0");
+    assert_eq!(kernel_version_number(&"5.10.0".into()), "5.10.0");
+}
+
+fn cpu_vendor(cpu: &Box<str>) -> &str {
+    let lowercase_cpu = cpu.to_ascii_lowercase();
+
+    if lowercase_cpu.contains("intel") {
+        "Intel"
+    } else if lowercase_cpu.contains("amd") {
+        "AMD"
+    } else if lowercase_cpu.contains("arm") {
+        "ARM"
+    } else if lowercase_cpu.contains("apple") {
+        "Apple"
+    } else {
+        "Other"
     }
 }
 
@@ -114,6 +149,7 @@ pub struct ChainStatsCollator {
     memory_memcpy_score: Counter<(u32, Option<u32>)>,
     disk_sequential_write_score: Counter<(u32, Option<u32>)>,
     disk_random_write_score: Counter<(u32, Option<u32>)>,
+    cpu_vendor: Counter<String>,
 }
 
 impl ChainStatsCollator {
@@ -148,7 +184,7 @@ impl ChainStatsCollator {
         self.linux_kernel.modify(
             sysinfo
                 .and_then(|sysinfo| sysinfo.linux_kernel.as_ref())
-                .map(|value| &**value),
+                .map(kernel_version_number),
             op,
         );
 
@@ -164,6 +200,11 @@ impl ChainStatsCollator {
             op,
         );
 
+        self.cpu_vendor.modify(
+            sysinfo.and_then(|sysinfo| sysinfo.cpu.as_ref().map(cpu_vendor)),
+            op,
+        );
+
         self.update_hwbench(hwbench, op);
     }
 
@@ -220,6 +261,7 @@ impl ChainStatsCollator {
                 .disk_sequential_write_score
                 .generate_ranking_ordered(),
             disk_random_write_score: self.disk_random_write_score.generate_ranking_ordered(),
+            cpu_vendor: self.cpu_vendor.generate_ranking_top(10),
         }
     }
 }

backend/telemetry_core/src/state/counter.rs

@@ -17,15 +17,15 @@
 use crate::feed_message::Ranking;
 use std::collections::HashMap;
 
-/// A data structure which counts how many occurences of a given key we've seen.
+/// A data structure which counts how many occurrences of a given key we've seen.
 #[derive(Default)]
 pub struct Counter<K> {
-    /// A map containing the number of occurences of a given key.
+    /// A map containing the number of occurrences of a given key.
     ///
     /// If there are none then the entry is removed.
     map: HashMap<K, u64>,
 
-    /// The number of occurences where the key is `None`.
+    /// The number of occurrences where the key is `None`.
     empty: u64,
 }