Stats with Opentelemetry

.env_file

@@ -14,3 +14,9 @@ MON_PORT=8765
 YDB_ENDPOINT=grpc://localhost:2136
 YDB_DATABASE=/local
 YDB_ANONYMOUS_CREDENTIALS=1
+CLICKHOUSE_DB=test
+CLICKHOUSE_USER=username
+CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1
+CLICKHOUSE_PASSWORD=pass
+SUPERSET_USERNAME=superset
+SUPERSET_PASSWORD=superset

.gitattributes

@@ -1,4 +1,6 @@
+
 *   text=auto
+*.png   binary
+*.jpg   binary
 *.py    text eol=lf
-*.sh    text eol=lf
-*   text working-tree-encoding=UTF-8
+*.sh    text eol=lf

.github/workflows/update_dashboard.yml

@@ -0,0 +1,45 @@
+name: update_dashboard
+
+on:
+  push:
+    branches:
+      - 'dev'
+    paths:
+      - 'dff/utils/docker/dockerfile_stats'
+      - 'dff/utils/docker/entrypoint_stats.sh'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  build_and_publish_dashboard:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+        - name: Checkout
+          uses: actions/checkout@v3
+
+        - name: Log in to container registry
+          uses: docker/login-action@v2
+          with:
+            registry: ghcr.io
+            username: ${{ github.actor }}
+            password: ${{ secrets.GITHUB_TOKEN }}
+
+        - name: Extract Docker metadata
+          uses: docker/metadata-action@v4
+          with:
+            images: ghcr.io/${{ github.repository }}/superset_df_dashboard
+            tags: |
+              type=ref,event=branch
+              type=semver,event={{version}}
+
+        - name: Build and upload image
+          uses: docker/build-push-action@v4 
+          with:
+            file: dff/utils/docker/dockerfile_stats
+            tags: ${{ steps.meta.outputs.tags }}
+            labels: ${{ steps.meta.outputs.labels }}

MANIFEST.in

@@ -1,6 +1,7 @@
 include CONTRIBUTING.md
 include LICENSE
 include README.md
+graft dff/config/superset_dashboard
 include dff/context_storages/protocols.json
 
 exclude makefile

dff/config/README.md

@@ -0,0 +1,10 @@
+# Superset dashboard config
+
+## Description
+
+This directory provides yaml files for Superset dashboard configuration.
+The files inside are not supposed to be edited manually for lest of compatibility breaks.
+Placeholders inside the files will be filled automatically when you use the 
+`dff.stats` CLI command to generate a configuration archive.
+
+Use `dff.stats -h` for more info.

dff/config/superset_dashboard/charts/Flow_visit_ratio_monitor_1.yaml

@@ -0,0 +1,78 @@
+slice_name: Flow visit ratio monitor
+description: null
+certified_by: null
+certification_details: null
+viz_type: echarts_timeseries_bar
+params:
+  datasource: 2__table
+  viz_type: echarts_timeseries_bar
+  slice_id: 1
+  granularity_sqla: start_time
+  time_grain_sqla: null
+  time_range: No filter
+  metrics:
+  - aggregate: COUNT
+    column:
+      advanced_data_type: null
+      certification_details: null
+      certified_by: null
+      column_name: context_id
+      description: null
+      expression: null
+      filterable: true
+      groupby: true
+      id: 1
+      is_certified: false
+      is_dttm: false
+      python_date_format: null
+      type: STRING
+      type_generic: 1
+      verbose_name: null
+      warning_markdown: null
+    expressionType: SIMPLE
+    hasCustomLabel: false
+    isNew: false
+    label: COUNT(context_id)
+    optionName: metric_waxxrrnkwwm_zudaex5z8bh
+    sqlExpression: null
+  groupby:
+  - flow_label
+  contributionMode: column
+  adhoc_filters: []
+  order_desc: true
+  row_limit: 10000
+  truncate_metric: true
+  show_empty_columns: true
+  comparison_type: values
+  annotation_layers: []
+  forecastPeriods: 10
+  forecastInterval: 0.8
+  orientation: vertical
+  x_axis_title_margin: 15
+  y_axis_title: ''
+  y_axis_title_margin: 50
+  y_axis_title_position: Left
+  color_scheme: supersetColors
+  stack: true
+  only_total: true
+  zoomable: true
+  show_legend: true
+  legendType: scroll
+  legendOrientation: top
+  x_axis_time_format: smart_date
+  y_axis_format: SMART_NUMBER
+  y_axis_bounds:
+  - null
+  - null
+  rich_tooltip: true
+  tooltipTimeFormat: smart_date
+  extra_form_data: {}
+  dashboards:
+  - 1
+query_context: '{"datasource":{"id":2,"type":"table"},"force":false,"queries":[{"time_range":"No
+  filter","granularity":"start_time","filters":[],"extras":{"time_grain_sqla":null,"having":"","where":""},"applied_time_extras":{},"columns":["flow_label"],"metrics":[{"aggregate":"COUNT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"context_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":1,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"STRING","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":false,"isNew":false,"label":"COUNT(context_id)","optionName":"metric_waxxrrnkwwm_zudaex5z8bh","sqlExpression":null}],"orderby":[[{"aggregate":"COUNT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"context_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":1,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"STRING","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":false,"isNew":false,"label":"COUNT(context_id)","optionName":"metric_waxxrrnkwwm_zudaex5z8bh","sqlExpression":null},false]],"annotation_layers":[],"row_limit":10000,"series_columns":["flow_label"],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"is_timeseries":true,"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["__timestamp"],"columns":["flow_label"],"aggregates":{"COUNT(context_id)":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"rename","options":{"columns":{"COUNT(context_id)":null},"level":0,"inplace":true}},{"operation":"contribution","options":{"orientation":"column"}},{"operation":"flatten"}]}],"form_data":{"datasource":"2__table","viz_type":"echarts_timeseries_bar","slice_id":1,"granularity_sqla":"start_time","time_grain_sqla":null,"time_range":"No
+  filter","metrics":[{"aggregate":"COUNT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"context_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":1,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"STRING","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":false,"isNew":false,"label":"COUNT(context_id)","optionName":"metric_waxxrrnkwwm_zudaex5z8bh","sqlExpression":null}],"groupby":["flow_label"],"contributionMode":"column","adhoc_filters":[],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"orientation":"vertical","x_axis_title_margin":15,"y_axis_title":"","y_axis_title_margin":50,"y_axis_title_position":"Left","color_scheme":"supersetColors","stack":true,"only_total":true,"zoomable":true,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","y_axis_format":"SMART_NUMBER","y_axis_bounds":[null,null],"rich_tooltip":true,"tooltipTimeFormat":"smart_date","extra_form_data":{},"dashboards":[1],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
+cache_timeout: null
+uuid: ba02528b-184b-4304-b027-f2b7d9011ab0
+version: 1.0.0
+dataset_uuid: fda98ab8-f550-45f1-9ded-0113f3e67260

dff/config/superset_dashboard/charts/Node_Visits_2.yaml

@@ -0,0 +1,59 @@
+slice_name: Node Visits
+description: null
+certified_by: null
+certification_details: null
+viz_type: dist_bar
+params:
+  adhoc_filters: []
+  bottom_margin: auto
+  color_scheme: supersetColors
+  columns:
+  - label
+  datasource: 3__table
+  extra_form_data: {}
+  granularity_sqla: start_time
+  groupby:
+  - request_id
+  metrics:
+  - aggregate: COUNT
+    column:
+      advanced_data_type: null
+      certification_details: null
+      certified_by: null
+      column_name: context_id
+      description: null
+      expression: null
+      filterable: true
+      groupby: true
+      id: 20
+      is_certified: false
+      is_dttm: false
+      python_date_format: null
+      type: STRING
+      type_generic: 1
+      verbose_name: null
+      warning_markdown: null
+    expressionType: SIMPLE
+    hasCustomLabel: false
+    isNew: false
+    label: COUNT(context_id)
+    optionName: metric_l2mle87zvnb_nfqzdxmig1d
+    sqlExpression: null
+  order_desc: true
+  rich_tooltip: true
+  row_limit: 10000
+  show_legend: true
+  time_range: No filter
+  viz_type: dist_bar
+  x_axis_label: History id
+  x_ticks_layout: auto
+  y_axis_bounds:
+  - null
+  - null
+  y_axis_format: SMART_NUMBER
+  y_axis_label: Node visits
+query_context: null
+cache_timeout: null
+uuid: 44f4ab9d-5072-4926-a6ed-8615fb81b3d0
+version: 1.0.0
+dataset_uuid: fda98ab8-f550-45f1-9ded-0113f3e67260

dff/config/superset_dashboard/charts/Node_counts_3.yaml

@@ -0,0 +1,59 @@
+slice_name: Node counts
+description: null
+certified_by: null
+certification_details: null
+viz_type: dist_bar
+params:
+  adhoc_filters: []
+  bar_stacked: true
+  bottom_margin: auto
+  color_scheme: supersetColors
+  columns:
+  - flow_label
+  datasource: 1__table
+  extra_form_data: {}
+  granularity_sqla: start_time
+  groupby:
+  - label
+  metrics:
+  - aggregate: COUNT_DISTINCT
+    column:
+      advanced_data_type: null
+      certification_details: null
+      certified_by: null
+      column_name: context_id
+      description: null
+      expression: null
+      filterable: true
+      groupby: true
+      id: 1
+      is_certified: false
+      is_dttm: false
+      python_date_format: null
+      type: STRING
+      type_generic: 1
+      verbose_name: null
+      warning_markdown: null
+    expressionType: SIMPLE
+    hasCustomLabel: false
+    isNew: false
+    label: COUNT_DISTINCT(context_id)
+    optionName: metric_axee7fzlpu_upud0bdjv6
+    sqlExpression: null
+  order_bars: true
+  order_desc: true
+  rich_tooltip: true
+  row_limit: 10000
+  show_legend: true
+  time_range: No filter
+  viz_type: dist_bar
+  x_ticks_layout: auto
+  y_axis_bounds:
+  - null
+  - null
+  y_axis_format: SMART_NUMBER
+query_context: null
+cache_timeout: null
+uuid: 0c47c7b5-f500-46cb-97e3-9ebb637f0c8a
+version: 1.0.0
+dataset_uuid: fda98ab8-f550-45f1-9ded-0113f3e67260

dff/config/superset_dashboard/charts/Node_visit_ratio_monitor_4.yaml

@@ -0,0 +1,86 @@
+slice_name: Node visit ratio monitor
+description: null
+certified_by: null
+certification_details: null
+viz_type: echarts_timeseries_bar
+params:
+  datasource: 2__table
+  viz_type: echarts_timeseries_bar
+  slice_id: 4
+  granularity_sqla: start_time
+  time_grain_sqla: null
+  time_range: No filter
+  metrics:
+  - aggregate: COUNT
+    column:
+      advanced_data_type: null
+      certification_details: null
+      certified_by: null
+      column_name: context_id
+      description: null
+      expression: null
+      filterable: true
+      groupby: true
+      id: 1
+      is_certified: false
+      is_dttm: false
+      python_date_format: null
+      type: STRING
+      type_generic: 1
+      verbose_name: null
+      warning_markdown: null
+    expressionType: SIMPLE
+    hasCustomLabel: false
+    isNew: false
+    label: COUNT(context_id)
+    optionName: metric_9yefk3wj2g_5wbp61n0pyr
+    sqlExpression: null
+  groupby:
+  - flow_label
+  - node_label
+  contributionMode: column
+  adhoc_filters: []
+  order_desc: true
+  row_limit: 10000
+  truncate_metric: true
+  show_empty_columns: true
+  comparison_type: values
+  annotation_layers: []
+  forecastPeriods: 10
+  forecastInterval: 0.8
+  orientation: vertical
+  x_axis_title: Datetime
+  x_axis_title_margin: 30
+  y_axis_title: Node visit ratio
+  y_axis_title_margin: 50
+  y_axis_title_position: Left
+  color_scheme: supersetColors
+  stack: true
+  only_total: true
+  zoomable: true
+  show_legend: true
+  legendType: scroll
+  legendOrientation: top
+  x_axis_time_format: smart_date
+  xAxisLabelRotation: 45
+  y_axis_format: SMART_NUMBER
+  logAxis: false
+  minorSplitLine: false
+  truncateYAxis: false
+  y_axis_bounds:
+  - null
+  - null
+  rich_tooltip: true
+  tooltipSortByMetric: true
+  tooltipTimeFormat: smart_date
+  extra_form_data: {}
+  dashboards:
+  - 1
+query_context: '{"datasource":{"id":2,"type":"table"},"force":false,"queries":[{"time_range":"No
+  filter","granularity":"start_time","filters":[],"extras":{"time_grain_sqla":null,"having":"","where":""},"applied_time_extras":{},"columns":["flow_label","node_label"],"metrics":[{"aggregate":"COUNT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"context_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":1,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"STRING","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":false,"isNew":false,"label":"COUNT(context_id)","optionName":"metric_9yefk3wj2g_5wbp61n0pyr","sqlExpression":null}],"orderby":[[{"aggregate":"COUNT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"context_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":1,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"STRING","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":false,"isNew":false,"label":"COUNT(context_id)","optionName":"metric_9yefk3wj2g_5wbp61n0pyr","sqlExpression":null},false]],"annotation_layers":[],"row_limit":10000,"series_columns":["flow_label","node_label"],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"is_timeseries":true,"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["__timestamp"],"columns":["flow_label","node_label"],"aggregates":{"COUNT(context_id)":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"rename","options":{"columns":{"COUNT(context_id)":null},"level":0,"inplace":true}},{"operation":"contribution","options":{"orientation":"column"}},{"operation":"flatten"}]}],"form_data":{"datasource":"2__table","viz_type":"echarts_timeseries_bar","slice_id":4,"granularity_sqla":"start_time","time_grain_sqla":null,"time_range":"No
+  filter","metrics":[{"aggregate":"COUNT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"context_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":1,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"STRING","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":false,"isNew":false,"label":"COUNT(context_id)","optionName":"metric_9yefk3wj2g_5wbp61n0pyr","sqlExpression":null}],"groupby":["flow_label","node_label"],"contributionMode":"column","adhoc_filters":[],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"orientation":"vertical","x_axis_title":"Datetime","x_axis_title_margin":30,"y_axis_title":"Node
+  visit ratio","y_axis_title_margin":50,"y_axis_title_position":"Left","color_scheme":"supersetColors","stack":true,"only_total":true,"zoomable":true,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","xAxisLabelRotation":45,"y_axis_format":"SMART_NUMBER","logAxis":false,"minorSplitLine":false,"truncateYAxis":false,"y_axis_bounds":[null,null],"rich_tooltip":true,"tooltipSortByMetric":true,"tooltipTimeFormat":"smart_date","extra_form_data":{},"dashboards":[1],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
+cache_timeout: null
+uuid: 6fafe59c-0fec-4cd8-a8b3-c0bfaffb2135
+version: 1.0.0
+dataset_uuid: fda98ab8-f550-45f1-9ded-0113f3e67260