V1.16.0 dev (#71)

* multi-region-usecase params

* add test script for development

* Update scenario scripts for exploring phase

* Fix pre-clean condition

* Update candidate model api

* add multi-src-as-te params

* update params for multi_src_as_te

* Fix diff2csv

* update multi_src_as_te params/flows

* update .env, configs/mddo-bgp, repos for multi_src_as_te usecase

---------

Co-authored-by: mddo <mddo>

.env

@@ -13,10 +13,10 @@ STATE_CONDUCTOR_IMAGE=ghcr.io/ool-mddo/state-conductor
 API_PROXY_IMAGE_TAG=1.21
 BATFISH_IMAGE_TAG=v0.3.0
 BATFISH_WRAPPER_IMAGE_TAG=v1.1.1
-BGP_POLICY_PARSER_IMAGE_TAG=v0.6.1
+BGP_POLICY_PARSER_IMAGE_TAG=v0.7.0
 FISH_TRACER_IMAGE_TAG=v1.0.0
-MODEL_CONDUCTOR_IMAGE_TAG=v1.11.0
-NETOMOX_EXP_IMAGE_TAG=v1.12.0
+MODEL_CONDUCTOR_IMAGE_TAG=v1.12.0
+NETOMOX_EXP_IMAGE_TAG=v1.13.0
 NETOVIZ_IMAGE_TAG=v0.7.0
 STATE_CONDUCTOR_IMAGE_TAG=v0.4.0
 

assets/api-proxy/default.conf

@@ -26,9 +26,9 @@ server {
     location /bgp_policy/ {
         proxy_pass http://bgp-policy-parser:5000;
     }
-    location /state-conductor/ {
-        proxy_pass http://state-conductor:5000;
-    }
+#    location /state-conductor/ {
+#        proxy_pass http://state-conductor:5000;
+#    }
     location / {
         proxy_pass http://fish-tracer:3000;
     }

demo/candidate_model_ops/00_run_phase.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/bash
+
+print_usage() {
+  echo "Usage: $(basename "$0") [options]"
+  echo "Options:"
+  echo "  -b     Benchmark topology name (default: original_asis)"
+  echo "  -c     Number of candidate topology to generate (default: 2)"
+  echo "  -d     Debug/data check, without executing ansible-runner (clab)"
+  echo "  -p     Phase number (default: 1)"
+  echo "  -u     Debug/step until (default: 99)"
+  echo "  -h     Display this help message"
+}
+
+# option check
+# defaults
+WITH_CLAB=true
+original_benchmark_topology=original_asis
+candidate_num=2
+phase=1
+step_until=99
+while getopts b:c:dp:u:h option; do
+  case $option in
+  b)
+    original_benchmark_topology="$OPTARG"
+    ;;
+  c)
+    candidate_num="$OPTARG"
+    ;;
+  d)
+    # data check, debug
+    # -> without container lab; does not build emulated-env
+    WITH_CLAB=false
+    ;;
+  p)
+    phase="$OPTARG"
+    ;;
+  u)
+    step_until="$OPTARG"
+    ;;
+  h)
+    print_usage
+    exit 0
+    ;;
+  *)
+    echo "Unknown option detected, -$OPTARG" >&2
+    print_usage
+    exit 1
+    ;;
+  esac
+done
+
+echo # newline
+echo "# check: phase = $phase"
+echo "# check: benchmark topology = $original_benchmark_topology"
+echo "# check: candidate number = $candidate_num"
+echo "# check: with_clab = $WITH_CLAB"
+echo "# check: step_until = $step_until"
+echo # newline
+
+# pre-clean (phase/session data)
+if [ "$phase" -eq 1 ]; then
+  bash phase_pre_clean.sh
+fi
+
+# generate candidate topologies
+if [ "$step_until" -ge 1 ]; then
+  bash 01_candidate_topology.sh -p "$phase" -c "$candidate_num" -b "$original_benchmark_topology"
+fi
+
+# Boot emulated env of benchmark topology (usually phase 1 and for original_asis only)
+if [ "$phase" -eq 1 ] && [ "$step_until" -ge 2 ]; then
+  if [ "$WITH_CLAB" == true ]; then
+    bash 02_benchmark_env.sh -p "$phase" -b "$original_benchmark_topology"
+  else
+    bash 02_benchmark_env.sh -p "$phase" -b "$original_benchmark_topology" -d # debug
+  fi
+fi
+
+# Boot each emulated env for candidate topology
+if [ "$step_until" -ge 3 ]; then
+  if [ "$WITH_CLAB" == true ]; then
+    bash 03_candidate_env.sh -p "$phase" -b "$original_benchmark_topology"
+  else
+    bash 03_candidate_env.sh -p "$phase" -b "$original_benchmark_topology" -d # debug
+  fi
+fi

demo/candidate_model_ops/01_candidate_topology.sh

@@ -2,64 +2,70 @@
 
 # shellcheck disable=SC1091
 source ./demo_vars
+# shellcheck disable=SC1091
+source ./orig_ns_topology.sh
 
-# Create original as-is topology data
-curl -s -X DELETE "http://${API_PROXY}/conduct/${NETWORK_NAME}"
-curl -s -X POST -H 'Content-Type: application/json' \
-  -d '{ "label": "original_asis", "phy_ss_only": true }' \
-  "http://${API_PROXY}/conduct/${NETWORK_NAME}/original_asis/topology"
+print_usage() {
+  echo "Usage: $(basename "$0") [options]"
+  echo "Options:"
+  echo "  -b     Benchmark topology name (default: original_asis)"
+  echo "  -c     Number of candidate topology to generate (default: 2)"
+  echo "  -p     Phase number (default: 1)"
+  echo "  -h     Display this help message"
+}
 
-if use_bgp_proc "$NETWORK_NAME" original_asis ; then
-  echo # newline
-  echo "Network:$NETWORK_NAME uses BGP, expand external-AS network and splice it into topology data"
+# option check
+# defaults
+original_benchmark_topology=original_asis
+candidate_num=2
+phase=1
+while getopts b:c:p:h option; do
+  case $option in
+  b)
+    original_benchmark_topology="$OPTARG"
+    ;;
+  c)
+    candidate_num="$OPTARG"
+    ;;
+  p)
+    phase="$OPTARG"
+    ;;
+  h)
+    print_usage
+    exit 0
+    ;;
+  *)
+    echo "Unknown option detected, -$OPTARG" >&2
+    print_usage
+    exit 1
+    ;;
+  esac
+done
 
-  # bgp-policy data handling
-  # parse configuration files with TTP
-  curl -s -X POST -H "Content-Type: application/json" \
-    -d '{}' \
-    "http://${API_PROXY}/bgp_policy/${NETWORK_NAME}/original_asis/parsed_result"
+echo # newline
+echo "# check: phase = $phase"
+echo "# check: benchmark topology = $original_benchmark_topology"
+echo "# check: candidate number = $candidate_num"
+echo # newline
 
-  # post bgp policy data to model-conductor to merge it with topology data
-  curl -s -X POST -H "Content-Type: application/json" \
-    -d '{}' \
-    "http://${API_PROXY}/bgp_policy/${NETWORK_NAME}/original_asis/topology"
+if [ "$phase" -eq 1 ] && [ "$original_benchmark_topology" == "original_asis" ]; then
+  # Create original as-is topology data
+  generate_original_asis_topology "$NETWORK_NAME"
 
-  # generate external-AS topology
-  external_as_json="${USECASE_SESSION_DIR}/external_as_topology.json"
-  curl -s "http://${API_PROXY}/usecases/${USECASE_NAME}/${NETWORK_NAME}/original_asis/external_as_topology?flow_data=event" \
-    > "$external_as_json"
+  # Splice external-AS topology to original as-is topology
+  splice_external_as_topology "$USECASE_NAME" "$NETWORK_NAME"
+fi
 
-  # splice external-AS topology to original_asis (overwrite)
-  curl -s -X POST -H "Content-Type: application/json" \
-    -d @<(jq '{ "overwrite": true, "ext_topology_data": . }' "$external_as_json") \
-    "http://${API_PROXY}/conduct/${NETWORK_NAME}/original_asis/splice_topology" \
-    > /dev/null # ignore echo-back (topology json)
+# convert benchmark topology name if specified emulated namespace topology
+if [[ $phase -ge 2 && $original_benchmark_topology == emulated_* ]]; then
+  original_benchmark_topology=$(reverse_snapshot_name "$original_benchmark_topology")
+  echo "# check: (reverse) benchmark topology = $original_benchmark_topology"
 fi
 
 # Generate candidate topologies
-original_candidate_list="${USECASE_SESSION_DIR}/original_candidate_list.json"
-curl -s -X POST -H 'Content-Type: application/json' \
-  -d '{
-    "candidate_number": "'"$CANDIDATE_NUM"'",
-    "usecase": {
-      "name": "'"$USECASE_NAME"'",
-      "sources": ["params", "flows/event"]
-    }
-  }' \
-  "http://${API_PROXY}/conduct/${NETWORK_NAME}/original_asis/candidate_topology" \
-  > "$original_candidate_list"
+generate_original_candidate_topologies "$USECASE_NAME" "$NETWORK_NAME" "$original_benchmark_topology" "$phase" "$candidate_num"
 
 # Add netoviz index
-netoviz_asis_index="${USECASE_SESSION_DIR}/netoviz_asis_index.json"
-jq '.[0:1]' "$NETWORK_INDEX" > "$netoviz_asis_index"
-netoviz_original_candidates_index="${USECASE_SESSION_DIR}/netoviz_original_candidates_index.json"
-filter='map(. + {label: ( "\(.network | ascii_upcase) (\(.snapshot))"), file: "topology.json"})'
-jq "$filter" "$original_candidate_list" > "$netoviz_original_candidates_index"
-netoviz_index="${USECASE_SESSION_DIR}/netoviz_index.json"
-jq -s '.[0] + .[1]' "$netoviz_asis_index" "$netoviz_original_candidates_index" > "$netoviz_index"
-
-curl -s -X POST -H 'Content-Type: application/json' \
-  -d @<(jq '{ "index_data": . }' "$netoviz_index") \
-  "http://${API_PROXY}/topologies/index"
+generate_netoviz_index "$phase" 1
 
 echo # newline

demo/candidate_model_ops/02_asis_env.sh → demo/candidate_model_ops/02_benchmark_env.sh

@@ -3,25 +3,36 @@
 # shellcheck disable=SC1091
 source ./demo_vars
 # shellcheck disable=SC1091
+source ./util.sh
+# shellcheck disable=SC1091
 source ./up_emulated_env.sh
 
 print_usage() {
   echo "Usage: $(basename "$0") [options]"
   echo "Options:"
+  echo "  -b     Benchmark topology name (default: original_asis)"
   echo "  -d     Debug/data check, without executing ansible-runner (clab)"
+  echo "  -p     Phase number (default: 1)"
   echo "  -h     Display this help message"
 }
 
 # option check
 # defaults
 WITH_CLAB=true
-while getopts dh option; do
+original_benchmark_topology=original_asis
+phase=1
+while getopts b:dp:h option; do
   case $option in
+  b)
+    original_benchmark_topology="$OPTARG"
+    ;;
   d)
     # data check, debug
     # -> without container lab; does not build emulated-env
     WITH_CLAB=false
-    echo "# Check: with_clab = $WITH_CLAB"
+    ;;
+  p)
+    phase="$OPTARG"
     ;;
   h)
     print_usage
@@ -35,25 +46,22 @@ while getopts dh option; do
   esac
 done
 
+echo # newline
+echo "# check: phase = $phase"
+echo "# check: benchmark topology = $original_benchmark_topology"
+echo "# check: with_clab = $WITH_CLAB"
+echo # newline
+
 # cache sudo credential
 echo "Please enter your sudo password:"
 sudo -v
 
 # at first: prepare emulated_asis topology data
-target_original_snapshot="original_asis"
-# convert namespace from original_asis to emulated_asis
-convert_namespace "$target_original_snapshot"
+# convert namespace from original namespace to emulated namespace
+convert_namespace "$original_benchmark_topology"
 
 # Add netoviz index
-netoviz_asis_index="${USECASE_SESSION_DIR}/netoviz_asis_index.json"
-jq '.[0:2]' "$NETWORK_INDEX" > "$netoviz_asis_index"
-netoviz_original_candidates_index="${USECASE_SESSION_DIR}/netoviz_original_candidates_index.json"
-netoviz_index="${USECASE_SESSION_DIR}/netoviz_index.json"
-jq -s '.[0] + .[1]' "$netoviz_asis_index" "$netoviz_original_candidates_index" > "$netoviz_index"
-
-curl -s -X POST -H 'Content-Type: application/json' \
-  -d @<(jq '{ "index_data": . }' "$netoviz_index") \
-  "http://${API_PROXY}/topologies/index"
+generate_netoviz_index "$phase" 2
 
 # up original_asis env
-up_emulated_env "$target_original_snapshot"
+up_emulated_env "$original_benchmark_topology"