From 0342b7c0f4a47aea246fef14a49eefc41dc41e3c Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Fri, 17 Nov 2023 12:37:59 -0800 Subject: [PATCH 01/49] pmumetrics - support for non-contiguous uncore device ids (#222) --- src/pmu2metrics/event_defs.go | 17 +++------ src/pmu2metrics/metadata.go | 64 ++++++++++++---------------------- src/pmu2metrics/metric_defs.go | 2 +- 3 files changed, 28 insertions(+), 55 deletions(-) diff --git a/src/pmu2metrics/event_defs.go b/src/pmu2metrics/event_defs.go index 6f3dcb6..2216abd 100644 --- a/src/pmu2metrics/event_defs.go +++ b/src/pmu2metrics/event_defs.go @@ -22,7 +22,7 @@ type GroupDefinition []EventDefinition // AKA a "group", ordered list of event d func isUncoreSupported(metadata Metadata) (supported bool) { supported = false - for uncoreDeviceName := range metadata.DeviceCounts { + for uncoreDeviceName := range metadata.DeviceIDs { if uncoreDeviceName == "cha" { // could be any uncore device supported = true break @@ -49,7 +49,7 @@ func isCollectableEvent(event EventDefinition, metadata Metadata) (collectable b // exclude uncore events when their corresponding device is not found if event.Device != "cpu" && event.Device != "" { deviceExists := false - for uncoreDeviceName := range metadata.DeviceCounts { + for uncoreDeviceName := range metadata.DeviceIDs { if event.Device == uncoreDeviceName { deviceExists = true break @@ -131,22 +131,13 @@ func expandUncoreGroups(groups []GroupDefinition, metadata Metadata) (expandedGr device := group[0].Device if device == "cha" || device == "upi" || device == "imc" || device == "iio" { var newGroups []GroupDefinition - // unlike other device types, imc ids may not be consecutive - var ids []int - if device == "imc" { - ids = metadata.IMCDeviceIDs - } else { - for i := 0; i < metadata.DeviceCounts[device]; i++ { - ids = append(ids, i) - } - } - if len(ids) == 0 { + if len(metadata.DeviceIDs[device]) == 0 { if gCmdLineArgs.verbose { log.Printf("No uncore devices found for %s", device) } continue } - if newGroups, err = expandUncoreGroup(group, ids, re); err != nil { + if newGroups, err = expandUncoreGroup(group, metadata.DeviceIDs[device], re); err != nil { return } expandedGroups = append(expandedGroups, newGroups...) diff --git a/src/pmu2metrics/metadata.go b/src/pmu2metrics/metadata.go index e09d11a..38375a0 100644 --- a/src/pmu2metrics/metadata.go +++ b/src/pmu2metrics/metadata.go @@ -18,8 +18,7 @@ import ( type Metadata struct { CoresPerSocket int `yaml:"CoresPerSocket"` - DeviceCounts map[string]int - IMCDeviceIDs []int + DeviceIDs map[string][]int Microarchitecture string `yaml:"Microarchitecture"` ModelName string PerfSupportedEvents string @@ -32,11 +31,7 @@ type Metadata struct { } func (md Metadata) String() string { - var uncoreDeviceCounts []string - for deviceType := range md.DeviceCounts { - uncoreDeviceCounts = append(uncoreDeviceCounts, fmt.Sprintf("%s: %d", deviceType, md.DeviceCounts[deviceType])) - } - return fmt.Sprintf(""+ + out := fmt.Sprintf(""+ "Model Name: %s, "+ "Microarchitecture: %s, "+ "Socket Count: %d, "+ @@ -44,9 +39,8 @@ func (md Metadata) String() string { "Threads per Core: %d, "+ "TSC Frequency (Hz): %d, "+ "TSC: %d, "+ - "Uncore Device Counts: %s, "+ "ref-cycles supported: %t, "+ - "TMA events supported: %t", + "TMA events supported: %t, ", md.ModelName, md.Microarchitecture, md.SocketCount, @@ -54,42 +48,34 @@ func (md Metadata) String() string { md.ThreadsPerCore, md.TSCFrequencyHz, md.TSC, - strings.Join(uncoreDeviceCounts, ", "), md.RefCyclesSupported, md.TMASupported) -} - -// counts uncore device files of format "uncore__id" in /sys/devices -func getDeviceCounts() (counts map[string]int, err error) { - counts = make(map[string]int) - var paths []string - pattern := filepath.Join("/", "sys", "devices", "uncore_*") - if paths, err = filepath.Glob(pattern); err != nil { - return - } - for _, path := range paths { - file := filepath.Base(path) - fields := strings.Split(file, "_") - if len(fields) == 3 { - counts[fields[1]] += 1 + for deviceName, deviceIds := range md.DeviceIDs { + var ids []string + for _, id := range deviceIds { + ids = append(ids, fmt.Sprintf("%d", id)) } + out += fmt.Sprintf("%s: [%s] ", deviceName, strings.Join(ids, ",")) } - return + return out } -func getIMCDeviceIds() (ids []int, err error) { - pattern := filepath.Join("/", "sys", "devices", "uncore_imc_*") - var files []string - if files, err = filepath.Glob(pattern); err != nil { +// /sys/bus/event_source/devices +func getUncoreDeviceIDs() (IDs map[string][]int, err error) { + pattern := filepath.Join("/", "sys", "bus", "event_source", "devices", "uncore_*") + var fileNames []string + if fileNames, err = filepath.Glob(pattern); err != nil { return } - re := regexp.MustCompile(`uncore_imc_(\d+)`) - for _, fileName := range files { + IDs = make(map[string][]int) + re := regexp.MustCompile(`uncore_(.*)_(\d+)`) + for _, fileName := range fileNames { match := re.FindStringSubmatch(fileName) - if match != nil { - id, _ := strconv.Atoi(match[1]) - ids = append(ids, id) + if match == nil { + continue } + id, _ := strconv.Atoi(match[2]) + IDs[match[1]] = append(IDs[match[1]], id) } return } @@ -293,12 +279,8 @@ func loadMetadata() (metadata Metadata, err error) { metadata.TSCFrequencyHz = GetTSCFreqMHz() * 1000000 // calculate TSC metadata.TSC = metadata.SocketCount * metadata.CoresPerSocket * metadata.ThreadsPerCore * metadata.TSCFrequencyHz - // /sys/devices counts - if metadata.DeviceCounts, err = getDeviceCounts(); err != nil { - return - } - // uncore imc device ids (may not be consecutive) - if metadata.IMCDeviceIDs, err = getIMCDeviceIds(); err != nil { + // uncore device IDs + if metadata.DeviceIDs, err = getUncoreDeviceIDs(); err != nil { return } // Model Name diff --git a/src/pmu2metrics/metric_defs.go b/src/pmu2metrics/metric_defs.go index 0c54adb..929aeac 100644 --- a/src/pmu2metrics/metric_defs.go +++ b/src/pmu2metrics/metric_defs.go @@ -165,7 +165,7 @@ func configureMetrics(metrics []MetricDefinition, evaluatorFunctions map[string] tscFreq := fmt.Sprintf("%f", float64(metadata.TSCFrequencyHz)) tsc := fmt.Sprintf("%f", float64(metadata.TSC)) coresPerSocket := fmt.Sprintf("%f", float64(metadata.CoresPerSocket)) - chasPerSocket := fmt.Sprintf("%f", float64(metadata.DeviceCounts["cha"])) + chasPerSocket := fmt.Sprintf("%f", float64(len(metadata.DeviceIDs["cha"]))) socketCount := fmt.Sprintf("%f", float64(metadata.SocketCount)) hyperThreadingOn := fmt.Sprintf("%t", metadata.ThreadsPerCore > 1) threadsPerCore := fmt.Sprintf("%f", float64(metadata.ThreadsPerCore)) From a28a4f43b6dcc718aa7586513d668feaa988211b Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Wed, 22 Nov 2023 12:46:52 -0800 Subject: [PATCH 02/49] refresh events and metrics to match perfspect (#223) --- src/pmu2metrics/resources/emr_events.txt | 22 +++-- src/pmu2metrics/resources/emr_metrics.json | 92 +++++++++++--------- src/pmu2metrics/resources/icx_events.txt | 19 +++-- src/pmu2metrics/resources/icx_metrics.json | 99 +++++++++------------- src/pmu2metrics/resources/spr_events.txt | 22 +++-- src/pmu2metrics/resources/spr_metrics.json | 92 +++++++++++--------- 6 files changed, 192 insertions(+), 154 deletions(-) diff --git a/src/pmu2metrics/resources/emr_events.txt b/src/pmu2metrics/resources/emr_events.txt index 92efcdb..5027aa0 100644 --- a/src/pmu2metrics/resources/emr_events.txt +++ b/src/pmu2metrics/resources/emr_events.txt @@ -57,10 +57,20 @@ cpu-cycles, ref-cycles, instructions; +cpu/event=0xcf,umask=0x10,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED2.512B_PACKED_HALF'/, +cpu/event=0xcf,umask=0x08,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED2.256B_PACKED_HALF'/, +cpu/event=0xc7,umask=0x80,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE'/, +cpu/event=0xc7,umask=0x40,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE'/, +cpu/event=0xc7,umask=0x20,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE'/, +cpu/event=0xc7,umask=0x10,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE'/, +cpu-cycles, +ref-cycles, +instructions; + cpu/event=0x47,umask=0x03,cmask=0x03,period=1000003,name='MEMORY_ACTIVITY.STALLS_L1D_MISS'/, +cpu/event=0x47,umask=0x05,cmask=0x05,period=1000003,name='MEMORY_ACTIVITY.STALLS_L2_MISS'/, cpu/event=0x12,umask=0x20,cmask=0x01,period=100003,name='DTLB_LOAD_MISSES.STLB_HIT:c1'/, cpu/event=0x12,umask=0x10,cmask=0x01,period=100003,name='DTLB_LOAD_MISSES.WALK_ACTIVE'/, -cpu/event=0x47,umask=0x05,cmask=0x05,period=1000003,name='MEMORY_ACTIVITY.STALLS_L2_MISS'/, cpu/event=0xa3,umask=0x10,cmask=0x10,period=1000003,name='CYCLE_ACTIVITY.CYCLES_MEM_ANY'/, cpu/event=0xb0,umask=0x09,cmask=0x01,period=1000003,name='ARITH.DIV_ACTIVE'/, cpu/event=0xad,umask=0x80,period=500009,name='INT_MISC.CLEAR_RESTEER_CYCLES'/, @@ -69,7 +79,6 @@ cpu-cycles, ref-cycles, instructions; -cpu/event=0xce,umask=0x02,cmask=0x01,period=100003,name='AMX_OPS_RETIRED.BF16:c1'/, cpu/event=0xd3,umask=0x10,cmask=0x00,period=100007,name='MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM'/, cpu/event=0xd1,umask=0x08,cmask=0x00,period=200003,name='MEM_LOAD_RETIRED.L1_MISS'/, cpu/event=0xd1,umask=0x80,cmask=0x00,period=1000003,name='MEM_LOAD_RETIRED.LOCAL_PMM'/, @@ -85,9 +94,7 @@ cpu/event=0xd0,umask=0x21,cmask=0x00,period=1000003,name='MEM_INST_RETIRED.LOCK_ cpu/event=0xd0,umask=0x82,cmask=0x00,period=1000003,name='MEM_INST_RETIRED.ALL_STORES'/, cpu/event=0x24,umask=0xe2,cmask=0x00,period=2000003,name='L2_RQSTS.ALL_RFO'/, cpu/event=0x24,umask=0xc2,cmask=0x00,period=2000003,name='L2_RQSTS.RFO_HIT'/, -cpu/event=0xcf,umask=0x03,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED2.SCALAR'/, cpu/event=0xcf,umask=0x1c,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED2.VECTOR'/, -cpu/event=0xc7,umask=0x03,period=100003,name='FP_ARITH_INST_RETIRED.SCALAR_SINGLE:u0x03'/, cpu/event=0xc7,umask=0x3c,period=100003,name='FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE:u0x3c'/, cpu-cycles, ref-cycles, @@ -105,6 +112,8 @@ cpu/event=0x79,umask=0x04,cmask=0x01,period=2000003,name='IDQ.MITE_CYCLES_ANY'/, cpu/event=0x79,umask=0x04,cmask=0x06,period=2000003,name='IDQ.MITE_CYCLES_OK'/, cpu/event=0x79,umask=0x08,cmask=0x01,period=2000003,name='IDQ.DSB_CYCLES_ANY'/, cpu/event=0x79,umask=0x08,cmask=0x06,period=2000003,name='IDQ.DSB_CYCLES_OK'/, +cpu/event=0xec,umask=0x02,period=2000003,name='CPU_CLK_UNHALTED.DISTRIBUTED'/, +cpu/event=0xb7,umask=0x02,period=2000003,name='EXE.AMX_BUSY'/, cpu-cycles, ref-cycles, instructions; @@ -133,6 +142,9 @@ cpu-cycles, ref-cycles, instructions; +cpu/event=0xe7,umask=0x0c,cmask=0x00,period=100003,name='INT_VEC_RETIRED.ADD_256'/, +cpu/event=0xe7,umask=0x20,cmask=0x00,period=100003,name='INT_VEC_RETIRED.VNNI_256'/, +cpu/event=0xe7,umask=0x80,cmask=0x00,period=100003,name='INT_VEC_RETIRED.MUL_256'/, cpu/event=0x79,umask=0x08,cmask=0x00,period=2000003,name='IDQ.DSB_UOPS'/, cpu/event=0x79,umask=0x04,period=100003,name='IDQ.MITE_UOPS'/, cpu/event=0x79,umask=0x20,period=100003,name='IDQ.MS_UOPS'/, @@ -183,4 +195,4 @@ imc/event=0x05,umask=0xf0,name='UNC_M_CAS_COUNT.WR'/; #power power/energy-pkg/, -power/energy-ram/; \ No newline at end of file +power/energy-ram/; diff --git a/src/pmu2metrics/resources/emr_metrics.json b/src/pmu2metrics/resources/emr_metrics.json index 76e803e..e81e6ed 100644 --- a/src/pmu2metrics/resources/emr_metrics.json +++ b/src/pmu2metrics/resources/emr_metrics.json @@ -86,6 +86,44 @@ "expression": "[L2_RQSTS.CODE_RD_MISS] / [instructions]", "expression-txn": "[L2_RQSTS.CODE_RD_MISS] / [TXN]" }, + { + "name": "metric_LLC code read MPI (demand+prefetch)", + "name-txn": "metric_LLC code read (demand+prefetch) misses per txn", + "expression": "[UNC_CHA_TOR_INSERTS.IA_MISS_CRD] / [instructions]", + "expression-txn": "[UNC_CHA_TOR_INSERTS.IA_MISS_CRD] / [TXN]" + }, + { + "name": "metric_LLC data read MPI (demand+prefetch)", + "name-txn": "metric_LLC data read (demand+prefetch) misses per txn", + "expression": "([UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF]) / [instructions]", + "expression-txn": "([UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF]) / [TXN]" + }, + { + "name": "metric_LLC total HITM (per instr) (excludes LLC prefetches)", + "name-txn": "metric_LLC total HITM per txn (excludes LLC prefetches)", + "expression": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM] / [instructions]", + "expression-txn": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM] / [TXN]", + "origin": "perfspect" + }, + { + "name": "metric_LLC total HIT clean line forwards (per instr) (excludes LLC prefetches)", + "name-txn": "metric_LLC total HIT clean line forwards per txn (excludes LLC prefetches)", + "expression": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD] / [instructions]", + "expression-txn": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD] / [TXN]", + "origin": "perfspect" + }, + { + "name": "metric_Average LLC demand data read miss latency (in ns)", + "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" + }, + { + "name": "metric_Average LLC demand data read miss latency for LOCAL requests (in ns)", + "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" + }, + { + "name": "metric_Average LLC demand data read miss latency for REMOTE requests (in ns)", + "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" + }, { "name": "metric_UPI Data transmit BW (MB/sec) (only data)", "expression": "([UNC_UPI_TxL_FLITS.ALL_DATA] * (64 / 9.0) / 1000000) / 1" @@ -140,44 +178,6 @@ "name": "metric_memory bandwidth total (MB/sec)", "expression": "(([UNC_M_CAS_COUNT.RD] + [UNC_M_CAS_COUNT.WR]) * 64 / 1000000) / 1" }, - { - "name": "metric_LLC code read MPI (demand+prefetch)", - "name-txn": "metric_LLC code read (demand+prefetch) misses per txn", - "expression": "[UNC_CHA_TOR_INSERTS.IA_MISS_CRD] / [instructions]", - "expression-txn": "[UNC_CHA_TOR_INSERTS.IA_MISS_CRD] / [TXN]" - }, - { - "name": "metric_LLC data read MPI (demand+prefetch)", - "name-txn": "metric_LLC data read (demand+prefetch) misses per txn", - "expression": "([UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF]) / [instructions]", - "expression-txn": "([UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF]) / [TXN]" - }, - { - "name": "metric_LLC total HITM (per instr) (excludes LLC prefetches)", - "name-txn": "metric_LLC total HITM per txn (excludes LLC prefetches)", - "expression": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM] / [instructions]", - "expression-txn": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM] / [TXN]", - "origin": "perfspect" - }, - { - "name": "metric_LLC total HIT clean line forwards (per instr) (excludes LLC prefetches)", - "name-txn": "metric_LLC total HIT clean line forwards per txn (excludes LLC prefetches)", - "expression": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD] / [instructions]", - "expression-txn": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD] / [TXN]", - "origin": "perfspect" - }, - { - "name": "metric_Average LLC demand data read miss latency (in ns)", - "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" - }, - { - "name": "metric_Average LLC demand data read miss latency for LOCAL requests (in ns)", - "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" - }, - { - "name": "metric_Average LLC demand data read miss latency for REMOTE requests (in ns)", - "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" - }, { "name": "metric_ITLB (2nd level) MPI", "name-txn": "metric_ITLB (2nd level) misses per txn", @@ -342,6 +342,10 @@ "name": "metric_TMA_......Ports_Utilized_0(%)", "expression": "100 * ( [EXE_ACTIVITY.3_PORTS_UTIL:u0x80] / ( [cpu-cycles] ) + ( [RESOURCE_STALLS.SCOREBOARD] / ( [cpu-cycles] ) ) * ( [CYCLE_ACTIVITY.STALLS_TOTAL] - [EXE_ACTIVITY.BOUND_ON_LOADS] ) / ( [cpu-cycles] ) )" }, + { + "name": "metric_TMA_........AMX_Busy(%)", + "expression": "100 * ( [EXE.AMX_BUSY] / ( [CPU_CLK_UNHALTED.DISTRIBUTED] ) )" + }, { "name": "metric_TMA_......Ports_Utilized_1(%)", "expression": "100 * ( [EXE_ACTIVITY.1_PORTS_UTIL] / ( [cpu-cycles] ) )" @@ -363,8 +367,16 @@ "expression": "100 * ( max( 0 , ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) - ( [PERF_METRICS.HEAVY_OPERATIONS] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) ) )" }, { - "name": "metric_TMA_....FP_Arith(%)", - "expression": "100 * ( ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * [UOPS_EXECUTED.X87] / [UOPS_EXECUTED.THREAD] ) + ( ( [FP_ARITH_INST_RETIRED.SCALAR_SINGLE:u0x03] + [FP_ARITH_INST_RETIRED2.SCALAR] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) + ( min( ( ( [FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE:u0x3c] + [FP_ARITH_INST_RETIRED2.VECTOR] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) , ( 1 ) ) ) + ( [AMX_OPS_RETIRED.BF16:c1] / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) )" + "name": "metric_TMA_........FP_Vector_256b(%)", + "expression": "100 * ( min( ( ( [FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE] + [FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE] + [FP_ARITH_INST_RETIRED2.256B_PACKED_HALF] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) , ( 1 ) ) )" + }, + { + "name": "metric_TMA_........FP_Vector_512b(%)", + "expression": "100 * ( min( ( ( [FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE] + [FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE] + [FP_ARITH_INST_RETIRED2.512B_PACKED_HALF] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) , ( 1 ) ) )" + }, + { + "name": "metric_TMA_......Int_Vector_256b(%)", + "expression": "100 * ( ( [INT_VEC_RETIRED.ADD_256] + [INT_VEC_RETIRED.MUL_256] + [INT_VEC_RETIRED.VNNI_256] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) )" }, { "name": "metric_TMA_..Heavy_Operations(%)", diff --git a/src/pmu2metrics/resources/icx_events.txt b/src/pmu2metrics/resources/icx_events.txt index 7e0e5ae..4d3457b 100644 --- a/src/pmu2metrics/resources/icx_events.txt +++ b/src/pmu2metrics/resources/icx_events.txt @@ -30,7 +30,6 @@ cpu-cycles, ref-cycles, instructions; -# OCR group 1 (ICX PMU supports a maximum of two OCR counters per group) cpu/event=0xb7,umask=0x01,offcore_rsp=0x104000477,name='OCR.READS_TO_CORE.LOCAL_DRAM'/, cpu/event=0xb7,umask=0x01,offcore_rsp=0x84002380,name='OCR.HWPF_L3.L3_MISS_LOCAL'/, cpu/event=0x85,umask=0x0e,period=100003,name='ITLB_MISSES.WALK_COMPLETED'/, @@ -39,7 +38,6 @@ cpu-cycles, ref-cycles, instructions; -# OCR group 2 (ICX PMU supports a maximum of two OCR counters per group) cpu/event=0xb7,umask=0x01,offcore_rsp=0x730000477,name='OCR.READS_TO_CORE.REMOTE_DRAM'/, cpu/event=0xb7,umask=0x01,offcore_rsp=0x90002380,name='OCR.HWPF_L3.REMOTE'/, cpu/event=0x08,umask=0x04,period=100003,name='DTLB_LOAD_MISSES.WALK_COMPLETED_2M_4M'/, @@ -69,6 +67,17 @@ cpu-cycles, ref-cycles, instructions; +#TMA AVX512 related +cpu/event=0xc7,umask=0x80,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE'/, +cpu/event=0xc7,umask=0x40,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE'/, +cpu/event=0xc7,umask=0x20,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE'/, +cpu/event=0xc7,umask=0x10,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE'/, +cpu/event=0xc7,umask=0x08,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE'/, +cpu/event=0xc7,umask=0x04,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE'/, +cpu-cycles, +ref-cycles, +instructions; + cpu/event=0x79,umask=0x04,cmask=0x01,period=2000003,name='IDQ.MITE_CYCLES_ANY'/, cpu/event=0x79,umask=0x04,cmask=0x05,period=2000003,name='IDQ.MITE_CYCLES_OK'/, cpu/event=0x79,umask=0x08,cmask=0x01,period=2000003,name='IDQ.DSB_CYCLES_ANY'/, @@ -129,10 +138,6 @@ cpu/event=0xd3,umask=0x02,cmask=0x00,period=100007,name='MEM_LOAD_L3_MISS_RETIRE cpu/event=0xd3,umask=0x01,cmask=0x00,period=100007,name='MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM'/, cpu/event=0xd3,umask=0x08,cmask=0x00,period=100007,name='MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD'/, cpu/event=0xd3,umask=0x04,cmask=0x00,period=100007,name='MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM'/, -cpu/event=0xb1,umask=0x10,cmask=0x00,period=2000003,name='UOPS_EXECUTED.X87'/, -cpu/event=0xb1,umask=0x01,cmask=0x00,period=2000003,name='UOPS_EXECUTED.THREAD'/, -cpu/event=0xc7,umask=0x03,period=100003,name='FP_ARITH_INST_RETIRED.SCALAR_SINGLE:u0x03'/, -cpu/event=0xc7,umask=0xfc,period=100003,name='FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE:u0xfc'/, cpu-cycles, ref-cycles, instructions; @@ -196,4 +201,4 @@ imc/event=0x04,umask=0x30,name='UNC_M_CAS_COUNT.WR'/; #power related power/energy-pkg/, -power/energy-ram/; \ No newline at end of file +power/energy-ram/; diff --git a/src/pmu2metrics/resources/icx_metrics.json b/src/pmu2metrics/resources/icx_metrics.json index 13ac99c..b63f7cb 100644 --- a/src/pmu2metrics/resources/icx_metrics.json +++ b/src/pmu2metrics/resources/icx_metrics.json @@ -86,6 +86,42 @@ "expression": "[L2_RQSTS.CODE_RD_MISS] / [instructions]", "expression-txn": "[L2_RQSTS.CODE_RD_MISS] / [TXN]" }, + { + "name": "metric_LLC code read MPI (demand+prefetch)", + "name-txn": "metric_LLC code read (demand+prefetch) misses per txn", + "expression": "([UNC_CHA_TOR_INSERTS.IA_MISS_CRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF]) / [instructions]", + "expression-txn": "([UNC_CHA_TOR_INSERTS.IA_MISS_CRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF]) / [TXN]" + }, + { + "name": "metric_LLC data read MPI (demand+prefetch)", + "name-txn": "metric_LLC data read (demand+prefetch) misses per txn", + "expression": "([UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF]) / [instructions]", + "expression-txn": "([UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF]) / [TXN]" + }, + { + "name": "metric_LLC total HITM (per instr) (excludes LLC prefetches)", + "name-txn": "metric_LLC total HITM per txn (excludes LLC prefetches)", + "expression": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM] / [instructions]", + "expression-txn": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM] / [TXN]" + }, + { + "name": "metric_LLC total HIT clean line forwards (per instr) (excludes LLC prefetches)", + "name-txn": "metric_LLC total HIT clean line forwards per txn (excludes LLC prefetches)", + "expression": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD] / [instructions]", + "expression-txn": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD] / [TXN]" + }, + { + "name": "metric_Average LLC demand data read miss latency (in ns)", + "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" + }, + { + "name": "metric_Average LLC demand data read miss latency for LOCAL requests (in ns)", + "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" + }, + { + "name": "metric_Average LLC demand data read miss latency for REMOTE requests (in ns)", + "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" + }, { "name": "metric_UPI Data transmit BW (MB/sec) (only data)", "expression": "([UNC_UPI_TxL_FLITS.ALL_DATA] * (64 / 9.0) / 1000000) / 1" @@ -118,21 +154,6 @@ "name": "metric_% Uops delivered from legacy decode pipeline (MITE)", "expression": "100 * ([IDQ.MITE_UOPS] / ([IDQ.DSB_UOPS] + [IDQ.MITE_UOPS] + [IDQ.MS_UOPS] + [LSD.UOPS]) )" }, - { - "name": "metric_core % cycles in non AVX license", - "expression": "(100 * [CORE_POWER.LVL0_TURBO_LICENSE]) / ([CORE_POWER.LVL0_TURBO_LICENSE] + [CORE_POWER.LVL1_TURBO_LICENSE] + [CORE_POWER.LVL2_TURBO_LICENSE])", - "origin": "perfspect" - }, - { - "name": "metric_core % cycles in AVX2 license", - "expression": "(100 * [CORE_POWER.LVL1_TURBO_LICENSE]) / ([CORE_POWER.LVL0_TURBO_LICENSE] + [CORE_POWER.LVL1_TURBO_LICENSE] + [CORE_POWER.LVL2_TURBO_LICENSE])", - "origin": "perfspect" - }, - { - "name": "metric_core % cycles in AVX-512 license", - "expression": "(100 * [CORE_POWER.LVL2_TURBO_LICENSE]) / ([CORE_POWER.LVL0_TURBO_LICENSE] + [CORE_POWER.LVL1_TURBO_LICENSE] + [CORE_POWER.LVL2_TURBO_LICENSE])", - "origin": "perfspect" - }, { "name": "metric_core initiated local dram read bandwidth (MB/sec)", "expression": "(([OCR.READS_TO_CORE.LOCAL_DRAM] + [OCR.HWPF_L3.L3_MISS_LOCAL]) * 64 / 1000000) / 1" @@ -153,42 +174,6 @@ "name": "metric_memory bandwidth total (MB/sec)", "expression": "(([UNC_M_CAS_COUNT.RD] + [UNC_M_CAS_COUNT.WR]) * 64 / 1000000) / 1" }, - { - "name": "metric_LLC code read MPI (demand+prefetch)", - "name-txn": "metric_LLC code read (demand+prefetch) misses per txn", - "expression": "([UNC_CHA_TOR_INSERTS.IA_MISS_CRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF]) / [instructions]", - "expression-txn": "([UNC_CHA_TOR_INSERTS.IA_MISS_CRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF]) / [TXN]" - }, - { - "name": "metric_LLC data read MPI (demand+prefetch)", - "name-txn": "metric_LLC data read (demand+prefetch) misses per txn", - "expression": "([UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF]) / [instructions]", - "expression-txn": "([UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF]) / [TXN]" - }, - { - "name": "metric_LLC total HITM (per instr) (excludes LLC prefetches)", - "name-txn": "metric_LLC total HITM per txn (excludes LLC prefetches)", - "expression": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM] / [instructions]", - "expression-txn": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM] / [TXN]" - }, - { - "name": "metric_LLC total HIT clean line forwards (per instr) (excludes LLC prefetches)", - "name-txn": "metric_LLC total HIT clean line forwards per txn (excludes LLC prefetches)", - "expression": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD] / [instructions]", - "expression-txn": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD] / [TXN]" - }, - { - "name": "metric_Average LLC demand data read miss latency (in ns)", - "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" - }, - { - "name": "metric_Average LLC demand data read miss latency for LOCAL requests (in ns)", - "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" - }, - { - "name": "metric_Average LLC demand data read miss latency for REMOTE requests (in ns)", - "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" - }, { "name": "metric_ITLB (2nd level) MPI", "name-txn": "metric_ITLB (2nd level) misses per txn", @@ -366,16 +351,16 @@ "expression": "100 * ( max( 0 , ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) - ( ( ( ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) / [UOPS_ISSUED.ANY] ) * [IDQ.MS_UOPS] / ( [TOPDOWN.SLOTS] ) ) + ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [UOPS_DECODED.DEC0] - [UOPS_DECODED.DEC0:c1] ) / [IDQ.MITE_UOPS] ) ) )" }, { - "name": "metric_TMA_....FP_Arith(%)", - "expression": "100 * ( ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * [UOPS_EXECUTED.X87] / [UOPS_EXECUTED.THREAD] ) + ( ( [FP_ARITH_INST_RETIRED.SCALAR_SINGLE:u0x03] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) + ( min( ( ( [FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE:u0xfc] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) , ( 1 ) ) ) )" + "name": "metric_TMA_........FP_Vector_128b(%)", + "expression": "100 * ( min( ( ( [FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE] + [FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) , ( 1 ) ) )" }, { - "name": "metric_TMA_......FP_Scalar(%)", - "expression": "100 * ( ( [FP_ARITH_INST_RETIRED.SCALAR_SINGLE:u0x03] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) )" + "name": "metric_TMA_........FP_Vector_256b(%)", + "expression": "100 * ( min( ( ( [FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE] + [FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) , ( 1 ) ) )" }, { - "name": "metric_TMA_......FP_Vector(%)", - "expression": "100 * ( min( ( ( [FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE:u0xfc] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) , ( 1 ) ) )" + "name": "metric_TMA_........FP_Vector_512b(%)", + "expression": "100 * ( min( ( ( [FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE] + [FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) , ( 1 ) ) )" }, { "name": "metric_TMA_..Heavy_Operations(%)", diff --git a/src/pmu2metrics/resources/spr_events.txt b/src/pmu2metrics/resources/spr_events.txt index 92efcdb..5027aa0 100644 --- a/src/pmu2metrics/resources/spr_events.txt +++ b/src/pmu2metrics/resources/spr_events.txt @@ -57,10 +57,20 @@ cpu-cycles, ref-cycles, instructions; +cpu/event=0xcf,umask=0x10,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED2.512B_PACKED_HALF'/, +cpu/event=0xcf,umask=0x08,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED2.256B_PACKED_HALF'/, +cpu/event=0xc7,umask=0x80,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE'/, +cpu/event=0xc7,umask=0x40,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE'/, +cpu/event=0xc7,umask=0x20,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE'/, +cpu/event=0xc7,umask=0x10,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE'/, +cpu-cycles, +ref-cycles, +instructions; + cpu/event=0x47,umask=0x03,cmask=0x03,period=1000003,name='MEMORY_ACTIVITY.STALLS_L1D_MISS'/, +cpu/event=0x47,umask=0x05,cmask=0x05,period=1000003,name='MEMORY_ACTIVITY.STALLS_L2_MISS'/, cpu/event=0x12,umask=0x20,cmask=0x01,period=100003,name='DTLB_LOAD_MISSES.STLB_HIT:c1'/, cpu/event=0x12,umask=0x10,cmask=0x01,period=100003,name='DTLB_LOAD_MISSES.WALK_ACTIVE'/, -cpu/event=0x47,umask=0x05,cmask=0x05,period=1000003,name='MEMORY_ACTIVITY.STALLS_L2_MISS'/, cpu/event=0xa3,umask=0x10,cmask=0x10,period=1000003,name='CYCLE_ACTIVITY.CYCLES_MEM_ANY'/, cpu/event=0xb0,umask=0x09,cmask=0x01,period=1000003,name='ARITH.DIV_ACTIVE'/, cpu/event=0xad,umask=0x80,period=500009,name='INT_MISC.CLEAR_RESTEER_CYCLES'/, @@ -69,7 +79,6 @@ cpu-cycles, ref-cycles, instructions; -cpu/event=0xce,umask=0x02,cmask=0x01,period=100003,name='AMX_OPS_RETIRED.BF16:c1'/, cpu/event=0xd3,umask=0x10,cmask=0x00,period=100007,name='MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM'/, cpu/event=0xd1,umask=0x08,cmask=0x00,period=200003,name='MEM_LOAD_RETIRED.L1_MISS'/, cpu/event=0xd1,umask=0x80,cmask=0x00,period=1000003,name='MEM_LOAD_RETIRED.LOCAL_PMM'/, @@ -85,9 +94,7 @@ cpu/event=0xd0,umask=0x21,cmask=0x00,period=1000003,name='MEM_INST_RETIRED.LOCK_ cpu/event=0xd0,umask=0x82,cmask=0x00,period=1000003,name='MEM_INST_RETIRED.ALL_STORES'/, cpu/event=0x24,umask=0xe2,cmask=0x00,period=2000003,name='L2_RQSTS.ALL_RFO'/, cpu/event=0x24,umask=0xc2,cmask=0x00,period=2000003,name='L2_RQSTS.RFO_HIT'/, -cpu/event=0xcf,umask=0x03,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED2.SCALAR'/, cpu/event=0xcf,umask=0x1c,cmask=0x00,period=100003,name='FP_ARITH_INST_RETIRED2.VECTOR'/, -cpu/event=0xc7,umask=0x03,period=100003,name='FP_ARITH_INST_RETIRED.SCALAR_SINGLE:u0x03'/, cpu/event=0xc7,umask=0x3c,period=100003,name='FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE:u0x3c'/, cpu-cycles, ref-cycles, @@ -105,6 +112,8 @@ cpu/event=0x79,umask=0x04,cmask=0x01,period=2000003,name='IDQ.MITE_CYCLES_ANY'/, cpu/event=0x79,umask=0x04,cmask=0x06,period=2000003,name='IDQ.MITE_CYCLES_OK'/, cpu/event=0x79,umask=0x08,cmask=0x01,period=2000003,name='IDQ.DSB_CYCLES_ANY'/, cpu/event=0x79,umask=0x08,cmask=0x06,period=2000003,name='IDQ.DSB_CYCLES_OK'/, +cpu/event=0xec,umask=0x02,period=2000003,name='CPU_CLK_UNHALTED.DISTRIBUTED'/, +cpu/event=0xb7,umask=0x02,period=2000003,name='EXE.AMX_BUSY'/, cpu-cycles, ref-cycles, instructions; @@ -133,6 +142,9 @@ cpu-cycles, ref-cycles, instructions; +cpu/event=0xe7,umask=0x0c,cmask=0x00,period=100003,name='INT_VEC_RETIRED.ADD_256'/, +cpu/event=0xe7,umask=0x20,cmask=0x00,period=100003,name='INT_VEC_RETIRED.VNNI_256'/, +cpu/event=0xe7,umask=0x80,cmask=0x00,period=100003,name='INT_VEC_RETIRED.MUL_256'/, cpu/event=0x79,umask=0x08,cmask=0x00,period=2000003,name='IDQ.DSB_UOPS'/, cpu/event=0x79,umask=0x04,period=100003,name='IDQ.MITE_UOPS'/, cpu/event=0x79,umask=0x20,period=100003,name='IDQ.MS_UOPS'/, @@ -183,4 +195,4 @@ imc/event=0x05,umask=0xf0,name='UNC_M_CAS_COUNT.WR'/; #power power/energy-pkg/, -power/energy-ram/; \ No newline at end of file +power/energy-ram/; diff --git a/src/pmu2metrics/resources/spr_metrics.json b/src/pmu2metrics/resources/spr_metrics.json index 76e803e..e81e6ed 100644 --- a/src/pmu2metrics/resources/spr_metrics.json +++ b/src/pmu2metrics/resources/spr_metrics.json @@ -86,6 +86,44 @@ "expression": "[L2_RQSTS.CODE_RD_MISS] / [instructions]", "expression-txn": "[L2_RQSTS.CODE_RD_MISS] / [TXN]" }, + { + "name": "metric_LLC code read MPI (demand+prefetch)", + "name-txn": "metric_LLC code read (demand+prefetch) misses per txn", + "expression": "[UNC_CHA_TOR_INSERTS.IA_MISS_CRD] / [instructions]", + "expression-txn": "[UNC_CHA_TOR_INSERTS.IA_MISS_CRD] / [TXN]" + }, + { + "name": "metric_LLC data read MPI (demand+prefetch)", + "name-txn": "metric_LLC data read (demand+prefetch) misses per txn", + "expression": "([UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF]) / [instructions]", + "expression-txn": "([UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF]) / [TXN]" + }, + { + "name": "metric_LLC total HITM (per instr) (excludes LLC prefetches)", + "name-txn": "metric_LLC total HITM per txn (excludes LLC prefetches)", + "expression": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM] / [instructions]", + "expression-txn": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM] / [TXN]", + "origin": "perfspect" + }, + { + "name": "metric_LLC total HIT clean line forwards (per instr) (excludes LLC prefetches)", + "name-txn": "metric_LLC total HIT clean line forwards per txn (excludes LLC prefetches)", + "expression": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD] / [instructions]", + "expression-txn": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD] / [TXN]", + "origin": "perfspect" + }, + { + "name": "metric_Average LLC demand data read miss latency (in ns)", + "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" + }, + { + "name": "metric_Average LLC demand data read miss latency for LOCAL requests (in ns)", + "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" + }, + { + "name": "metric_Average LLC demand data read miss latency for REMOTE requests (in ns)", + "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" + }, { "name": "metric_UPI Data transmit BW (MB/sec) (only data)", "expression": "([UNC_UPI_TxL_FLITS.ALL_DATA] * (64 / 9.0) / 1000000) / 1" @@ -140,44 +178,6 @@ "name": "metric_memory bandwidth total (MB/sec)", "expression": "(([UNC_M_CAS_COUNT.RD] + [UNC_M_CAS_COUNT.WR]) * 64 / 1000000) / 1" }, - { - "name": "metric_LLC code read MPI (demand+prefetch)", - "name-txn": "metric_LLC code read (demand+prefetch) misses per txn", - "expression": "[UNC_CHA_TOR_INSERTS.IA_MISS_CRD] / [instructions]", - "expression-txn": "[UNC_CHA_TOR_INSERTS.IA_MISS_CRD] / [TXN]" - }, - { - "name": "metric_LLC data read MPI (demand+prefetch)", - "name-txn": "metric_LLC data read (demand+prefetch) misses per txn", - "expression": "([UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF]) / [instructions]", - "expression-txn": "([UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD] + [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF]) / [TXN]" - }, - { - "name": "metric_LLC total HITM (per instr) (excludes LLC prefetches)", - "name-txn": "metric_LLC total HITM per txn (excludes LLC prefetches)", - "expression": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM] / [instructions]", - "expression-txn": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM] / [TXN]", - "origin": "perfspect" - }, - { - "name": "metric_LLC total HIT clean line forwards (per instr) (excludes LLC prefetches)", - "name-txn": "metric_LLC total HIT clean line forwards per txn (excludes LLC prefetches)", - "expression": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD] / [instructions]", - "expression-txn": "[OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD] / [TXN]", - "origin": "perfspect" - }, - { - "name": "metric_Average LLC demand data read miss latency (in ns)", - "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" - }, - { - "name": "metric_Average LLC demand data read miss latency for LOCAL requests (in ns)", - "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" - }, - { - "name": "metric_Average LLC demand data read miss latency for REMOTE requests (in ns)", - "expression": "( 1000000000 * ([UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE] / [UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE]) / ([UNC_CHA_CLOCKTICKS] / ([CHAS_PER_SOCKET] * [SOCKET_COUNT]) ) ) * 1" - }, { "name": "metric_ITLB (2nd level) MPI", "name-txn": "metric_ITLB (2nd level) misses per txn", @@ -342,6 +342,10 @@ "name": "metric_TMA_......Ports_Utilized_0(%)", "expression": "100 * ( [EXE_ACTIVITY.3_PORTS_UTIL:u0x80] / ( [cpu-cycles] ) + ( [RESOURCE_STALLS.SCOREBOARD] / ( [cpu-cycles] ) ) * ( [CYCLE_ACTIVITY.STALLS_TOTAL] - [EXE_ACTIVITY.BOUND_ON_LOADS] ) / ( [cpu-cycles] ) )" }, + { + "name": "metric_TMA_........AMX_Busy(%)", + "expression": "100 * ( [EXE.AMX_BUSY] / ( [CPU_CLK_UNHALTED.DISTRIBUTED] ) )" + }, { "name": "metric_TMA_......Ports_Utilized_1(%)", "expression": "100 * ( [EXE_ACTIVITY.1_PORTS_UTIL] / ( [cpu-cycles] ) )" @@ -363,8 +367,16 @@ "expression": "100 * ( max( 0 , ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) - ( [PERF_METRICS.HEAVY_OPERATIONS] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) ) )" }, { - "name": "metric_TMA_....FP_Arith(%)", - "expression": "100 * ( ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * [UOPS_EXECUTED.X87] / [UOPS_EXECUTED.THREAD] ) + ( ( [FP_ARITH_INST_RETIRED.SCALAR_SINGLE:u0x03] + [FP_ARITH_INST_RETIRED2.SCALAR] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) + ( min( ( ( [FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE:u0x3c] + [FP_ARITH_INST_RETIRED2.VECTOR] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) , ( 1 ) ) ) + ( [AMX_OPS_RETIRED.BF16:c1] / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) )" + "name": "metric_TMA_........FP_Vector_256b(%)", + "expression": "100 * ( min( ( ( [FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE] + [FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE] + [FP_ARITH_INST_RETIRED2.256B_PACKED_HALF] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) , ( 1 ) ) )" + }, + { + "name": "metric_TMA_........FP_Vector_512b(%)", + "expression": "100 * ( min( ( ( [FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE] + [FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE] + [FP_ARITH_INST_RETIRED2.512B_PACKED_HALF] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) ) , ( 1 ) ) )" + }, + { + "name": "metric_TMA_......Int_Vector_256b(%)", + "expression": "100 * ( ( [INT_VEC_RETIRED.ADD_256] + [INT_VEC_RETIRED.MUL_256] + [INT_VEC_RETIRED.VNNI_256] ) / ( ( [PERF_METRICS.RETIRING] / ( [PERF_METRICS.FRONTEND_BOUND] + [PERF_METRICS.BAD_SPECULATION] + [PERF_METRICS.RETIRING] + [PERF_METRICS.BACKEND_BOUND] ) ) * ( [TOPDOWN.SLOTS] ) ) )" }, { "name": "metric_TMA_..Heavy_Operations(%)", From eae076fa6f137061466d7539cd739361fe452ce9 Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Tue, 28 Nov 2023 07:20:52 -0800 Subject: [PATCH 03/49] add AMP prefetcher detection for new Xeons (#224) --- .../resources/collector_reports.yaml.tmpl | 2 +- src/reporter/report_tables.go | 2 +- src/reporter/source.go | 31 ++++++++++++++----- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/orchestrator/resources/collector_reports.yaml.tmpl b/src/orchestrator/resources/collector_reports.yaml.tmpl index b12c66f..a3265b4 100644 --- a/src/orchestrator/resources/collector_reports.yaml.tmpl +++ b/src/orchestrator/resources/collector_reports.yaml.tmpl @@ -127,7 +127,7 @@ commands: superuser: true parallel: true - label: rdmsr 0x1a4 - command: rdmsrx -f 3:0 0x1a4 # MSR_MISC_FEATURE_CONTROL: L2 and DCU Prefetcher enabled/disabled + command: rdmsrx -f 7:0 0x1a4 # MSR_PREFETCH_CONTROL: L2, DCU, and AMP Prefetchers enabled/disabled superuser: true modprobe: msr parallel: true diff --git a/src/reporter/report_tables.go b/src/reporter/report_tables.go index 550fd2c..3eb291c 100644 --- a/src/reporter/report_tables.go +++ b/src/reporter/report_tables.go @@ -769,7 +769,7 @@ func newCPUTable(sources []*Source, cpusInfo *cpu.CPU, category TableCategory) ( source.getL3(microarchitecture), source.getL3PerCore(microarchitecture, coresPerSocket, sockets, virtualization), channels, - source.getPrefetchers(), + source.getPrefetchers(microarchitecture), source.getTurboEnabled(family), virtualization, source.getPPINs(), diff --git a/src/reporter/source.go b/src/reporter/source.go index 6191d61..4648df2 100644 --- a/src/reporter/source.go +++ b/src/reporter/source.go @@ -612,23 +612,38 @@ func (s *Source) getL3PerCore(uArch string, coresPerSocketStr string, socketsStr return } -func (s *Source) getPrefetchers() (val string) { +func (s *Source) getPrefetchers(uarch string) (val string) { prefetchers := s.valFromRegexSubmatch("rdmsr 0x1a4", `^([0-9a-fA-F]+)`) if prefetchers != "" { prefetcherInt, err := strconv.ParseInt(prefetchers, 16, 64) if err == nil { + // MSR_PREFETCH_CONTROL // prefetchers are enabled when associated bit is 0 - // 1: "L2 HW" - // 2: "L2 Adj." - // 4: "DCU HW" - // 8: "DCU IP" + // bit 0 -- 1: "L2 HW" + // bit 1 -- 2: "L2 Adj." + // bit 2 -- 4: "DCU HW" + // bit 3 -- 8: "DCU IP" + // bit 4 -- reserved + // bit 5 -- 32: "AMP" - Adapative Multipath Probability (SPR, EMR, GNR) + // bit 6-63 -- reserved + prefNames := []string{"L2 HW", "L2 Adj.", "DCU HW", "DCU IP", ""} // all Xeons + if strings.Contains(uarch, "SPR") || strings.Contains(uarch, "EMR") || strings.Contains(uarch, "GNR") { + prefNames = append(prefNames, "AMP") + } var prefList []string - for i, pref := range []string{"L2 HW", "L2 Adj.", "DCU HW", "DCU IP"} { + for i, pref := range prefNames { + if pref == "" { + continue + } bitMask := int64(math.Pow(2, float64(i))) - // if bit is zero + var enabledDisabled string + // enabled if bit is zero if bitMask&prefetcherInt == 0 { - prefList = append(prefList, pref) + enabledDisabled = "Enabled" + } else { + enabledDisabled = "Disabled" } + prefList = append(prefList, fmt.Sprintf("%s: %s", pref, enabledDisabled)) } if len(prefList) > 0 { val = strings.Join(prefList, ", ") From b8d99e5a15de04314e68e0f4efdd2415e257c984 Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Wed, 29 Nov 2023 06:02:51 -0800 Subject: [PATCH 04/49] move prefetchers from feature table to CPU table (#227) --- src/reporter/report_tables.go | 2 - src/reporter/source.go | 74 ++++++++++++++++++----------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/reporter/report_tables.go b/src/reporter/report_tables.go index 3eb291c..336f0a2 100644 --- a/src/reporter/report_tables.go +++ b/src/reporter/report_tables.go @@ -2070,11 +2070,9 @@ func newFeatureTable(sources []*Source, category TableCategory) (table *Table) { ValueNames: []string{ "BI_2IFU_4_F_VICTIMS_EN", "EnableDBPForF", - "NoHmlessPref", "FBThreadSlicing", "DISABLE_FASTGO", "SpecI2MEn", - "disable_llpref", "DPT_DISABLE", }, } diff --git a/src/reporter/source.go b/src/reporter/source.go index 4648df2..084027a 100644 --- a/src/reporter/source.go +++ b/src/reporter/source.go @@ -613,45 +613,49 @@ func (s *Source) getL3PerCore(uArch string, coresPerSocketStr string, socketsStr } func (s *Source) getPrefetchers(uarch string) (val string) { - prefetchers := s.valFromRegexSubmatch("rdmsr 0x1a4", `^([0-9a-fA-F]+)`) - if prefetchers != "" { - prefetcherInt, err := strconv.ParseInt(prefetchers, 16, 64) - if err == nil { - // MSR_PREFETCH_CONTROL - // prefetchers are enabled when associated bit is 0 - // bit 0 -- 1: "L2 HW" - // bit 1 -- 2: "L2 Adj." - // bit 2 -- 4: "DCU HW" - // bit 3 -- 8: "DCU IP" - // bit 4 -- reserved - // bit 5 -- 32: "AMP" - Adapative Multipath Probability (SPR, EMR, GNR) - // bit 6-63 -- reserved - prefNames := []string{"L2 HW", "L2 Adj.", "DCU HW", "DCU IP", ""} // all Xeons - if strings.Contains(uarch, "SPR") || strings.Contains(uarch, "EMR") || strings.Contains(uarch, "GNR") { - prefNames = append(prefNames, "AMP") + // MSR_PREFETCH_CONTROL + // prefetchers are enabled when associated bit is 0 + prefetcherDefs := []struct { + name string + msr string + bit int + uarchs string + }{ + {name: "L2 HW", msr: "0x1a4", bit: 0, uarchs: "all"}, + {name: "L2 Adj.", msr: "0x1a4", bit: 1, uarchs: "all"}, + {name: "DCU HW", msr: "0x1a4", bit: 2, uarchs: "all"}, + {name: "DCU IP", msr: "0x1a4", bit: 3, uarchs: "all"}, + {name: "AMP", msr: "0x1a4", bit: 5, uarchs: "SPR_EMR_GNR"}, + {name: "Homeless", msr: "0x6d", bit: 14, uarchs: "SPR_EMR_GNR"}, + {name: "LLC", msr: "0x6d", bit: 42, uarchs: "SPR_EMR_GNR"}, + } + var prefList []string + for _, pf := range prefetcherDefs { + if pf.uarchs == "all" || strings.Contains(pf.uarchs, uarch[:3]) { + msrVal := s.valFromRegexSubmatch(fmt.Sprintf("rdmsr %s", pf.msr), `^([0-9a-fA-F]+)`) + if msrVal == "" { + continue } - var prefList []string - for i, pref := range prefNames { - if pref == "" { - continue - } - bitMask := int64(math.Pow(2, float64(i))) - var enabledDisabled string - // enabled if bit is zero - if bitMask&prefetcherInt == 0 { - enabledDisabled = "Enabled" - } else { - enabledDisabled = "Disabled" - } - prefList = append(prefList, fmt.Sprintf("%s: %s", pref, enabledDisabled)) + msrInt, err := strconv.ParseInt(msrVal, 16, 64) + if err != nil { + continue } - if len(prefList) > 0 { - val = strings.Join(prefList, ", ") + bitMask := int64(math.Pow(2, float64(pf.bit))) + var enabledDisabled string + // enabled if bit is zero + if bitMask&msrInt == 0 { + enabledDisabled = "Enabled" } else { - val = "None" + enabledDisabled = "Disabled" } + prefList = append(prefList, fmt.Sprintf("%s: %s", pf.name, enabledDisabled)) } } + if len(prefList) > 0 { + val = strings.Join(prefList, ", ") + } else { + val = "None" + } return } @@ -659,11 +663,9 @@ func (s *Source) getPrefetchers(uarch string) (val string) { .................... bit default "BI to IFU", 2 0 "EnableDBPForF", 3 0 -"NoHmlessPref", 14 0 "DisFBThreadSlicing", 15 1 "DISABLE_FASTGO", 27 0 "SpecI2MEn", 30 1 -"disable_llpref", 42 0 "DPT_DISABLE", 45 0 */ func (s *Source) getFeatures() (vals []string) { @@ -671,7 +673,7 @@ func (s *Source) getFeatures() (vals []string) { if features != "" { featureInt, err := strconv.ParseInt(features, 16, 64) if err == nil { - for _, bit := range []int{2, 3, 14, 15, 27, 30, 42, 45} { + for _, bit := range []int{2, 3, 15, 27, 30, 45} { bitMask := int64(math.Pow(2, float64(bit))) vals = append(vals, fmt.Sprintf("%d", bitMask&featureInt>>bit)) } From 25f125bf8526d35da524db732a486c8dcba96573 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Nov 2023 06:04:10 -0800 Subject: [PATCH 05/49] Bump golang.org/x/term from 0.14.0 to 0.15.0 in /src/pkg/progress (#226) Bumps [golang.org/x/term](https://github.com/golang/term) from 0.14.0 to 0.15.0. - [Commits](https://github.com/golang/term/compare/v0.14.0...v0.15.0) --- src/pkg/progress/go.mod | 4 ++-- src/pkg/progress/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pkg/progress/go.mod b/src/pkg/progress/go.mod index 2c55a34..f6105db 100644 --- a/src/pkg/progress/go.mod +++ b/src/pkg/progress/go.mod @@ -2,6 +2,6 @@ module intel.com/svr-info/pkg/progress/v2 go 1.21 -require golang.org/x/term v0.14.0 +require golang.org/x/term v0.15.0 -require golang.org/x/sys v0.14.0 // indirect +require golang.org/x/sys v0.15.0 // indirect diff --git a/src/pkg/progress/go.sum b/src/pkg/progress/go.sum index c2b3f18..0871f56 100644 --- a/src/pkg/progress/go.sum +++ b/src/pkg/progress/go.sum @@ -1,4 +1,4 @@ -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= From 163c77080e7c19cf5b28d438acd67e87ba964e8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Nov 2023 06:04:37 -0800 Subject: [PATCH 06/49] Bump golang.org/x/term from 0.14.0 to 0.15.0 in /src/orchestrator (#225) Bumps [golang.org/x/term](https://github.com/golang/term) from 0.14.0 to 0.15.0. - [Commits](https://github.com/golang/term/compare/v0.14.0...v0.15.0) --- src/orchestrator/go.mod | 4 ++-- src/orchestrator/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/orchestrator/go.mod b/src/orchestrator/go.mod index b587306..41ef903 100644 --- a/src/orchestrator/go.mod +++ b/src/orchestrator/go.mod @@ -16,7 +16,7 @@ replace intel.com/svr-info/pkg/commandfile => ../pkg/commandfile require ( golang.org/x/exp v0.0.0-20231006140011-7918f672742d - golang.org/x/term v0.14.0 + golang.org/x/term v0.15.0 gopkg.in/yaml.v2 v2.4.0 intel.com/svr-info/pkg/commandfile v0.0.0-00010101000000-000000000000 intel.com/svr-info/pkg/core v0.0.0-00010101000000-000000000000 @@ -27,6 +27,6 @@ require ( require ( github.com/creasty/defaults v1.7.0 // indirect github.com/kr/pretty v0.3.1 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/sys v0.15.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/src/orchestrator/go.sum b/src/orchestrator/go.sum index 733f433..dd0e0ea 100644 --- a/src/orchestrator/go.sum +++ b/src/orchestrator/go.sum @@ -13,10 +13,10 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From d08b1dd4d968526624b59b0d72ac9edcbe579af7 Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Thu, 30 Nov 2023 16:12:23 -0800 Subject: [PATCH 07/49] set EMR MCC cache way count to (correct) 15 (#228) EMR MCC and XCC have different cache sizes/ways. --- src/reporter/source.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/reporter/source.go b/src/reporter/source.go index 084027a..55237a7 100644 --- a/src/reporter/source.go +++ b/src/reporter/source.go @@ -493,9 +493,9 @@ func (s *Source) getCacheWays(uArch string) (cacheWays []int64) { wayCount = 11 } else if uArch == "ICX" { wayCount = 12 - } else if uArch == "SPR_MCC" || uArch == "SPR_XCC" { + } else if uArch == "SPR_MCC" || uArch == "SPR_XCC" || uArch == "EMR_MCC" { wayCount = 15 - } else if uArch == "EMR_MCC" || uArch == "EMR_XCC" { + } else if uArch == "EMR_XCC" { wayCount = 20 } else { return From 77dd8474263e97557d7c4d88a0cf0a111694eac2 Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Mon, 11 Dec 2023 10:59:12 -0800 Subject: [PATCH 08/49] recognice Dell Inc. as vendor for memory population chart generation (#231) --- src/reporter/report_tables.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/reporter/report_tables.go b/src/reporter/report_tables.go index 336f0a2..06ff8f7 100644 --- a/src/reporter/report_tables.go +++ b/src/reporter/report_tables.go @@ -1317,7 +1317,7 @@ func newDIMMPopulationTable(sources []*Source, dimmTable *Table, cpusInfo *cpu.C } else { vendor := source.valFromDmiDecodeRegexSubmatch("0", `^\s*Vendor:\s*(.+?)$`) sockets, _ := strconv.Atoi(source.valFromRegexSubmatch("lscpu", `^Socket\(.*:\s*(.+?)$`)) - if vendor == "Dell" { + if strings.Contains(vendor, "Dell") { err := deriveDIMMInfoDell(&hv.Values, sockets, channels) if err != nil { log.Printf("%v", err) From cef752621730f310e48005ecb2ae3eb078a6e77f Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Mon, 11 Dec 2023 11:00:12 -0800 Subject: [PATCH 09/49] Handle very deep call stacks when analyzing. (#230) --- src/reporter/go.mod | 2 +- src/reporter/go.sum | 4 ++-- src/reporter/process_stacks.go | 13 +++++++------ src/reporter/report_generator_html.go | 6 ++++++ src/reporter/source.go | 8 ++++++-- 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/reporter/go.mod b/src/reporter/go.mod index 46f57c2..8240afa 100644 --- a/src/reporter/go.mod +++ b/src/reporter/go.mod @@ -45,7 +45,7 @@ require ( golang.org/x/crypto v0.14.0 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/tools v0.14.0 // indirect gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect gopkg.in/src-d/go-git.v4 v4.13.1 // indirect diff --git a/src/reporter/go.sum b/src/reporter/go.sum index 6be31db..437e6a0 100644 --- a/src/reporter/go.sum +++ b/src/reporter/go.sum @@ -140,8 +140,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/src/reporter/process_stacks.go b/src/reporter/process_stacks.go index 7bbcd8b..6fcf89c 100644 --- a/src/reporter/process_stacks.go +++ b/src/reporter/process_stacks.go @@ -45,14 +45,13 @@ func (p *ProcessStacks) parsePerfFolded(folded string) (err error) { } func (p *ProcessStacks) parseAsyncProfilerFolded(folded string, processName string) (err error) { - re := regexp.MustCompile(`^(.+) (\d+)$`) for _, line := range strings.Split(folded, "\n") { - match := re.FindStringSubmatch(line) - if match == nil { + splitAt := strings.LastIndex(line, " ") + if splitAt == -1 { continue } - stack := match[1] - count, err := strconv.Atoi(match[2]) + stack := line[:splitAt] + count, err := strconv.Atoi(line[splitAt+1:]) if err != nil { continue } @@ -98,11 +97,13 @@ func (p *ProcessStacks) averageDepth(processName string) (average float64) { } func (p *ProcessStacks) dumpFolded() (folded string) { + var sb strings.Builder for processName, stacks := range *p { for stack, stackCount := range stacks { - folded += fmt.Sprintf("%s;%s %d\n", processName, stack, stackCount) + sb.WriteString(fmt.Sprintf("%s;%s %d\n", processName, stack, stackCount)) } } + folded = sb.String() return } diff --git a/src/reporter/report_generator_html.go b/src/reporter/report_generator_html.go index 872aa74..f6d3d82 100644 --- a/src/reporter/report_generator_html.go +++ b/src/reporter/report_generator_html.go @@ -1078,6 +1078,8 @@ func (n *Node) MarshalJSON() ([]byte, error) { }) } +var maxStackDepth = 50 + func convertFoldedToJSON(folded string) (out string, err error) { rootNode := Node{Name: "root", Value: 0, Children: make(map[string]*Node)} scanner := bufio.NewScanner(strings.NewReader(folded)) @@ -1088,6 +1090,10 @@ func convertFoldedToJSON(folded string) (out string, err error) { v := line[sep+1:] stack := strings.Split(s, ";") reverse(stack) + if len(stack) > maxStackDepth { + log.Printf("Trimming call stack depth from %d to %d", len(stack), maxStackDepth) + stack = stack[:maxStackDepth] + } var i int i, err = strconv.Atoi(v) if err != nil { diff --git a/src/reporter/source.go b/src/reporter/source.go index 55237a7..b680c7c 100644 --- a/src/reporter/source.go +++ b/src/reporter/source.go @@ -1001,12 +1001,16 @@ func (s *Source) getJavaFolded() (folded string) { re := regexp.MustCompile(`^async-profiler (\d+) (.*)$`) for header, stacks := range asyncProfilerOutput { if stacks == "" { - log.Printf("no stacks for: %s", header) + log.Printf("profiling data error, no stacks for: %s", header) + continue + } + if strings.HasPrefix(stacks, "Failed to inject profiler") { + log.Printf("profiling data error: %s", stacks) continue } match := re.FindStringSubmatch(header) if match == nil { - log.Printf("header didn't match regex: %s", header) + log.Printf("profiling data error, regex didn't match header: %s", header) continue } pid := match[1] From c913cef1bef732ee8bb0cc25935046186634bc00 Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Mon, 11 Dec 2023 11:01:02 -0800 Subject: [PATCH 10/49] add per-process mode to pmu2metrics (#229) --- go.work.sum | 5 +- src/pmu2metrics/event_defs.go | 17 +- src/pmu2metrics/go.sum | 1 + src/pmu2metrics/main.go | 448 ++++++++++++++++++++++++---------- src/pmu2metrics/metrics.go | 51 ++-- src/pmu2metrics/process.go | 101 ++++++++ src/reporter/source.go | 5 +- 7 files changed, 471 insertions(+), 157 deletions(-) create mode 100644 src/pmu2metrics/process.go diff --git a/go.work.sum b/go.work.sum index 6239b4a..1c9e977 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,9 +1,10 @@ -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/hyperjumptech/hyper-mux v1.1.0/go.mod h1:qdok3j+/VEtFvJ+YOotTNskGg2BXg3UJTbycU2xFDvE= github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/xuri/excelize/v2 v2.7.0 h1:Hri/czwyRCW6f6zrCDWXcXKshlq4xAZNpNOpdfnFhEw= github.com/xuri/excelize/v2 v2.7.0/go.mod h1:ebKlRoS+rGyLMyUx3ErBECXs/HNYqyj+PbkkKRK5vSI= golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= @@ -20,10 +21,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= diff --git a/src/pmu2metrics/event_defs.go b/src/pmu2metrics/event_defs.go index 2216abd..f18ae04 100644 --- a/src/pmu2metrics/event_defs.go +++ b/src/pmu2metrics/event_defs.go @@ -43,11 +43,17 @@ func isCollectableEvent(event EventDefinition, metadata Metadata) (collectable b return } // short-circuit off-core response events - if event.Device == "cpu" && strings.HasPrefix(event.Name, "OCR") && isUncoreSupported(metadata) { + if event.Device == "cpu" && strings.HasPrefix(event.Name, "OCR") && isUncoreSupported(metadata) && !gCmdLineArgs.processMode { return } - // exclude uncore events when their corresponding device is not found + // exclude uncore events when + // - their corresponding device is not found + // - not in system-wide collection mode if event.Device != "cpu" && event.Device != "" { + if gCmdLineArgs.processMode { + collectable = false + return + } deviceExists := false for uncoreDeviceName := range metadata.DeviceIDs { if event.Device == uncoreDeviceName { @@ -69,7 +75,12 @@ func isCollectableEvent(event EventDefinition, metadata Metadata) (collectable b return } // no uncore means we're on a VM where cpu fixed cycles are likely not supported - if !isUncoreSupported(metadata) && strings.Contains(event.Name, "cpu-cycles") { + if strings.Contains(event.Name, "cpu-cycles") && !isUncoreSupported(metadata) { + collectable = false + return + } + // no cstate and power events in process mode + if gCmdLineArgs.processMode && (strings.Contains(event.Name, "cstate_") || strings.Contains(event.Name, "power/energy")) { collectable = false return } diff --git a/src/pmu2metrics/go.sum b/src/pmu2metrics/go.sum index 051dc9d..16b9ca3 100644 --- a/src/pmu2metrics/go.sum +++ b/src/pmu2metrics/go.sum @@ -4,6 +4,7 @@ github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17 github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/src/pmu2metrics/main.go b/src/pmu2metrics/main.go index 5f3d160..0be8978 100644 --- a/src/pmu2metrics/main.go +++ b/src/pmu2metrics/main.go @@ -16,21 +16,30 @@ import ( ) type CmdLineArgs struct { - showHelp bool - showVersion bool - timeout int // seconds + showHelp bool + showVersion bool + showMetricNames bool + timeout int // seconds + // process options + processMode bool + pidList string + processFilter string + processCount int + processRefresh int // secons + // advanced options eventFilePath string metricFilePath string perfPrintInterval int // milliseconds perfMuxInterval int // milliseconds - printCSV bool - verbose bool - veryVerbose bool - metadataFilePath string - perfStatFilePath string - showMetricNames bool - metricsList string - printWide bool + // output format options + metricsList string + printWide bool + printCSV bool + verbose bool + veryVerbose bool + // debugging options + metadataFilePath string + perfStatFilePath string } // globals @@ -96,9 +105,99 @@ func getPerfPath() (path string, tempDir string, err error) { return } +func printMetrics(process Process, metrics []Metric, frameCount int, frameTimestamp float64) { + if gCmdLineArgs.printCSV { + if frameCount == 1 { + // print "Timestamp,PID,CMD,", then metric names as CSV headers + fmt.Print("Timestamp,PID,CMD,") + var names []string + for _, metric := range metrics { + names = append(names, metric.Name) + } + fmt.Printf("%s\n", strings.Join(names, ",")) + } + fmt.Printf("%d,%s,%s,", gCollectionStartTime.Unix()+int64(frameTimestamp), process.pid, process.comm) + var values []string + for _, metric := range metrics { + values = append(values, strconv.FormatFloat(metric.Value, 'g', 8, 64)) + } + fmt.Printf("%s\n", strings.Join(values, ",")) + } else { // human readable output + if !gCmdLineArgs.printWide { + fmt.Println("--------------------------------------------------------------------------------------") + fmt.Printf("- Metrics captured at %s\n", gCollectionStartTime.Add(time.Second*time.Duration(int(frameTimestamp))).UTC()) + if process.pid != "" { + fmt.Printf("- PID: %s\n", process.pid) + fmt.Printf("- CMD: %s\n", process.comm) + } + fmt.Println("--------------------------------------------------------------------------------------") + fmt.Printf("%-70s %15s\n", "metric", "value") + fmt.Printf("%-70s %15s\n", "------------------------", "----------") + for _, metric := range metrics { + fmt.Printf("%-70s %15s\n", metric.Name, strconv.FormatFloat(metric.Value, 'g', 4, 64)) + } + } else { // wide format + var names []string + var values []float64 + for _, metric := range metrics { + names = append(names, metric.Name) + values = append(values, metric.Value) + } + minColWidth := 6 + colSpacing := 3 + if frameCount == 1 { // print headers + header := "Timestamp " // 10 + 3 + if process.pid != "" { + header += "PID " // 7 + 3 + header += "Command " // 15 + 3 + } + for _, name := range names { + extend := 0 + if len(name) < minColWidth { + extend = minColWidth - len(name) + } + header += fmt.Sprintf("%s%*s%*s", name, extend, "", colSpacing, "") + } + fmt.Println(header) + } + // handle values + TimestampColWidth := 10 + formattedTimestamp := fmt.Sprintf("%d", gCollectionStartTime.Unix()+int64(frameTimestamp)) + row := fmt.Sprintf("%s%*s%*s", formattedTimestamp, TimestampColWidth-len(formattedTimestamp), "", colSpacing, "") + if process.pid != "" { + PIDColWidth := 7 + commandColWidth := 15 + row += fmt.Sprintf("%s%*s%*s", process.pid, PIDColWidth-len(process.pid), "", colSpacing, "") + var command string + if len(process.comm) <= commandColWidth { + command = process.comm + } else { + command = process.comm[:commandColWidth] + } + row += fmt.Sprintf("%s%*s%*s", command, commandColWidth-len(command), "", colSpacing, "") + } + // handle the metric values + for i, value := range values { + colWidth := max(len(names[i]), minColWidth) + formattedVal := fmt.Sprintf("%.2f", value) + row += fmt.Sprintf("%s%*s%*s", formattedVal, colWidth-len(formattedVal), "", colSpacing, "") + } + fmt.Println(row) + } + } +} + // build perf args from event groups -func getPerfCommandArgs(eventGroups []GroupDefinition, metadata Metadata) (args []string, err error) { - args = append(args, []string{"stat", "-I", fmt.Sprintf("%d", gCmdLineArgs.perfPrintInterval), "-a", "-j", "-e"}...) +func getPerfCommandArgs(pid string, timeout int, eventGroups []GroupDefinition, metadata Metadata) (args []string, err error) { + args = append(args, "stat", "-I", fmt.Sprintf("%d", gCmdLineArgs.perfPrintInterval), "-j") + // add pid, if applicable + if pid != "" { + args = append(args, "-p", pid) + } else { + args = append(args, "-a") // system-wide collection + } + // add events to collect + args = append(args, "-e") var groups []string for _, group := range eventGroups { var events []string @@ -108,34 +207,87 @@ func getPerfCommandArgs(eventGroups []GroupDefinition, metadata Metadata) (args groups = append(groups, fmt.Sprintf("{%s}", strings.Join(events, ","))) } args = append(args, fmt.Sprintf("'%s'", strings.Join(groups, ","))) - if gCmdLineArgs.timeout > 0 { - args = append(args, "sleep") - args = append(args, fmt.Sprintf("%d", gCmdLineArgs.timeout)) + // add timeout, if applicable + if timeout != 0 { + args = append(args, "sleep", fmt.Sprintf("%d", timeout)) } return } -func doWork(perfPath string, eventGroups []GroupDefinition, metricDefinitions []MetricDefinition, metadata Metadata) (err error) { - var args []string - if args, err = getPerfCommandArgs(eventGroups, metadata); err != nil { - log.Printf("failed to assemble perf args: %v", err) - return +func getPerfCommands(perfPath string, eventGroups []GroupDefinition, metadata Metadata) (processes []Process, perfCommands []*exec.Cmd, err error) { + if gCmdLineArgs.processMode { + if gCmdLineArgs.pidList != "" { + pids := strings.Split(gCmdLineArgs.pidList, ",") + for _, pid := range pids { + if processExists(pid) { + var process Process + if process, err = getProcess(pid); err != nil { + return + } + processes = append(processes, process) + } + } + } else { + if processes, err = getHotProcesses(gCmdLineArgs.processCount, gCmdLineArgs.processFilter); err != nil { + return + } + } + if len(processes) == 0 { + err = fmt.Errorf("no PIDs selected") + return + } + var timeout int + if gCmdLineArgs.timeout > 0 && gCmdLineArgs.timeout < gCmdLineArgs.processRefresh { + timeout = gCmdLineArgs.timeout + } else { + timeout = gCmdLineArgs.processRefresh + } + for _, process := range processes { + var args []string + if args, err = getPerfCommandArgs(process.pid, timeout, eventGroups, metadata); err != nil { + err = fmt.Errorf("failed to assemble perf args: %v", err) + return + } + cmd := exec.Command(perfPath, args...) + perfCommands = append(perfCommands, cmd) + } + } else { + var args []string + if args, err = getPerfCommandArgs("", gCmdLineArgs.timeout, eventGroups, metadata); err != nil { + err = fmt.Errorf("failed to assemble perf args: %v", err) + return + } + cmd := exec.Command(perfPath, args...) + perfCommands = append(perfCommands, cmd) } - cmd := exec.Command(perfPath, args...) + return +} + +// MetricFrame -- the metrics values and associated metadata +type MetricFrame struct { + process Process + metrics []Metric + frameCount int + timestamp float64 +} + +func runPerf(process Process, cmd *exec.Cmd, metricDefinitions []MetricDefinition, metadata Metadata, frameChannel chan MetricFrame, errorChannel chan error) { + var err error + defer func() { errorChannel <- err }() reader, _ := cmd.StderrPipe() if gCmdLineArgs.veryVerbose { + log.Printf("PID: %s", process.pid) log.Print(cmd) } scanner := bufio.NewScanner(reader) var outputLines [][]byte - // start perf stat + // start perf if err = cmd.Start(); err != nil { - log.Printf("failed to run perf: %v", err) + err = fmt.Errorf("failed to run perf: %v", err) + log.Printf("%v", err) return } - // get current time for use in setting timestamps on output - gCollectionStartTime = time.Now() - // Use a timer to determine when to send a frame of events back to the caller (over the eventChannel). + // Use a timer to determine when we received an entire frame of events from perf // The timer will expire when no lines (events) have been received from perf for more than 100ms. This // works because perf writes the events to stderr in a burst every collection interval, e.g., 5 seconds. // When the timer expires, this code assumes that perf is done writing events to stderr. @@ -153,13 +305,13 @@ func doWork(perfPath string, eventGroups []GroupDefinition, metricDefinitions [] return } frameCount += 1 - printMetrics(metrics, frameCount, frameTimestamp) + frameChannel <- MetricFrame{process, metrics, frameCount, frameTimestamp} outputLines = [][]byte{} // empty it } } }() - // read perf stat output - for scanner.Scan() { // blocks waiting for next token (line) + // read perf output + for scanner.Scan() { // blocks waiting for next token (line), loop terminated (Scan returns false) when file empty/closed line := scanner.Text() if gCmdLineArgs.veryVerbose { log.Print(line) @@ -175,23 +327,75 @@ func doWork(perfPath string, eventGroups []GroupDefinition, metricDefinitions [] return } frameCount += 1 - printMetrics(metrics, frameCount, frameTimestamp) + frameChannel <- MetricFrame{process, metrics, frameCount, frameTimestamp} } // wait for perf stat to exit if err = cmd.Wait(); err != nil { - if strings.Contains(err.Error(), "signal") { + if strings.Contains(err.Error(), "signal") { // perf received kill signal, ignore err = nil } else { - log.Printf("error from perf stat on exit: %v", err) + err = fmt.Errorf("error from perf on exit: %v", err) + log.Printf("%v", err) } return } +} + +func receiveMetrics(frameChannel chan MetricFrame) { + totalFrameCount := 0 + for frame := range frameChannel { + totalFrameCount++ + printMetrics(frame.process, frame.metrics, totalFrameCount, frame.timestamp) + } +} + +func doWork(perfPath string, eventGroups []GroupDefinition, metricDefinitions []MetricDefinition, metadata Metadata) (err error) { + refreshPIDs := gCmdLineArgs.processMode && gCmdLineArgs.pidList == "" + errorChannel := make(chan error) + frameChannel := make(chan MetricFrame) + totalRuntimeSeconds := 0 // only relevant in process Mode + go receiveMetrics(frameChannel) + for { + // get current time for use in setting timestamps on output + gCollectionStartTime = time.Now() + var perfCommands []*exec.Cmd + var processes []Process + if processes, perfCommands, err = getPerfCommands(perfPath, eventGroups, metadata); err != nil { + return + } + for i, cmd := range perfCommands { + var process Process + if len(processes) > i { + process = processes[i] + } + go runPerf(process, cmd, metricDefinitions, metadata, frameChannel, errorChannel) + } + // wait for all runPerf goroutines to finish + for range perfCommands { + err = <-errorChannel // TODO: capture all errors + } + totalRuntimeSeconds += gCmdLineArgs.processRefresh + if !refreshPIDs || (gCmdLineArgs.timeout != 0 && totalRuntimeSeconds >= gCmdLineArgs.timeout) { + break + } + } + close(frameChannel) // trigger receiveMetrics to end return } // Function used for testing and debugging // Plays back events present in a file that contains perf stat output -func doWorkDebug(perfStatFilePath string, metricDefinitions []MetricDefinition, metadata Metadata) (err error) { +func doWorkDebug(perfStatFilePath string, eventGroups []GroupDefinition, metricDefinitions []MetricDefinition, metadata Metadata) (err error) { + gCollectionStartTime = time.Now() + var perfCommands []*exec.Cmd + var processes []Process + if processes, perfCommands, err = getPerfCommands("perf", nil /*eventGroups*/, metadata); err != nil { + return + } + for _, cmd := range perfCommands { + log.Print(cmd) + } + log.Print(processes) file, err := os.Open(perfStatFilePath) if err != nil { return @@ -220,7 +424,11 @@ func doWorkDebug(perfStatFilePath string, metricDefinitions []MetricDefinition, return } frameCount++ - printMetrics(metrics, frameCount, frameTimestamp) + var process Process + if gCmdLineArgs.processMode { + process = Process{pid: fmt.Sprintf("%d", frameCount), cmd: "long command", comm: "process name is big"} + } + printMetrics(process, metrics, frameCount, frameTimestamp) outputLines = [][]byte{} // empty it } } @@ -234,7 +442,11 @@ func doWorkDebug(perfStatFilePath string, metricDefinitions []MetricDefinition, return } frameCount += 1 - printMetrics(metrics, frameCount, frameTimestamp) + var process Process + if gCmdLineArgs.processMode { + process = Process{pid: fmt.Sprintf("%d", frameCount), cmd: "long command", comm: "process name is big"} + } + printMetrics(process, metrics, frameCount, frameTimestamp) } err = scanner.Err() return @@ -249,20 +461,34 @@ func showUsage() { usage := ` -h, --help Print this usage message and exit. - --csv - CSV formatted output. + -V, --version + Show program version and exit. --list Show metric names available on this platform and exit. + +Collection Options: + -t, --timeout + Number of seconds to run (default: indefinitely). + --per-process + Enable process mode. Associates metrics with processes. + -p, --pid + Comma separated list of pids. Only valid when in process mode (default: None). + --process-filter + Regular expression used to match process names. Valid only when in process mode and --pid not specified (default: None). + --process-count + The number of processes to monitor. Used only when in process mode and --pid not specified (default: 5). + --process-refesh + The number of seconds to run before refreshing the process list. Used only when in process mode and --pid not specified (default: 30). + +Output Options: --metrics Metric names to include in output. (Quoted and comma separated list.) + --csv + CSV formatted output. Best for parsing. --wide - Wide formatted output. Best when a few selected metrics are printed. - -t, --timeout - Number of seconds to run. By default, runs indefinitely. + Wide formatted output. Best used when a small number of metrics are printed. -v[v] - Enable verbose logging. - -V, --version - Show program version and exit. + Enable verbose, or very verbose (-vv) logging. Advanced Options: -e, --eventfile @@ -270,22 +496,22 @@ Advanced Options: -m, --metricfile Path to metric definition file. -i, --interval - Event collection interval in milliseconds + Event collection interval in milliseconds (default: 5000). -x, --muxinterval - Multiplexing interval in milliseconds` + Multiplexing interval in milliseconds (default: 125).` fmt.Println(usage) } func validateArgs() (err error) { if gCmdLineArgs.metadataFilePath != "" { if gCmdLineArgs.perfStatFilePath == "" { - err = fmt.Errorf("-p and -d options must both be specified") + err = fmt.Errorf("-perfstat and -metadata options must both be specified") return } } if gCmdLineArgs.perfStatFilePath != "" { if gCmdLineArgs.metadataFilePath == "" { - err = fmt.Errorf("-p and -d options must both be specified") + err = fmt.Errorf("-perfstat and -metadata options must both be specified") return } } @@ -293,71 +519,22 @@ func validateArgs() (err error) { err = fmt.Errorf("-csv and -wide are mutually exclusive, choose one") return } - return -} - -func printMetrics(metrics []Metric, frameCount int, frameTimestamp float64) { - if gCmdLineArgs.printCSV { - if frameCount == 1 { - // print "Timestamp,", then metric names as CSV headers - fmt.Print("Timestamp,") - var names []string - for _, metric := range metrics { - names = append(names, metric.Name) - } - fmt.Printf("%s\n", strings.Join(names, ",")) - } - fmt.Printf("%d,", gCollectionStartTime.Unix()+int64(frameTimestamp)) - var values []string - for _, metric := range metrics { - values = append(values, strconv.FormatFloat(metric.Value, 'g', 8, 64)) - } - fmt.Printf("%s\n", strings.Join(values, ",")) - } else { // human readable output - if !gCmdLineArgs.printWide { - fmt.Println("--------------------------------------------------------------------------------------") - fmt.Printf("- Metrics captured at %s\n", gCollectionStartTime.Add(time.Second*time.Duration(int(frameTimestamp))).UTC()) - fmt.Println("--------------------------------------------------------------------------------------") - fmt.Printf("%-70s %15s\n", "metric", "value") - fmt.Printf("%-70s %15s\n", "------------------------", "----------") - for _, metric := range metrics { - fmt.Printf("%-70s %15s\n", metric.Name, strconv.FormatFloat(metric.Value, 'g', 4, 64)) - } - } else { // wide format - var names []string - var values []float64 - for _, metric := range metrics { - names = append(names, metric.Name) - values = append(values, metric.Value) - } - if frameCount == 1 { // print headers - header := "Timestamp " - header += strings.Join(names, " ") - fmt.Printf("%s\n", header) - } - // handle timestamp - colWidth := 10 - colSpacing := 3 - val := fmt.Sprintf("%d", gCollectionStartTime.Unix()+int64(frameTimestamp)) - row := fmt.Sprintf("%s%*s%*s", val, colWidth-len(val), "", colSpacing, "") - // handle the metric values - for i, value := range values { - colWidth = len(names[i]) - val = fmt.Sprintf("%.2f", value) - row += fmt.Sprintf("%s%*s%*s", val, colWidth-len(val), "", colSpacing, "") - } - fmt.Println(row) - } + if gCmdLineArgs.pidList != "" && !gCmdLineArgs.processMode { + err = fmt.Errorf("-pid only valid in process mode") + return + } + if gCmdLineArgs.processFilter != "" && !gCmdLineArgs.processMode { + err = fmt.Errorf("-pfilter only valid in process mode") + return } + if gCmdLineArgs.pidList != "" && gCmdLineArgs.processFilter != "" { + err = fmt.Errorf("-pid and -pfilter are mutually exclusive") + return + } + return } -const ( - exitNoError = 0 - exitError = 1 - exitInterrupt = 2 -) - -func mainReturnWithCode() int { +func configureArgs() { flag.Usage = func() { showUsage() } // override default usage output flag.BoolVar(&gCmdLineArgs.showHelp, "h", false, "") flag.BoolVar(&gCmdLineArgs.showHelp, "help", false, "") @@ -365,8 +542,21 @@ func mainReturnWithCode() int { flag.BoolVar(&gCmdLineArgs.showVersion, "version", false, "") flag.BoolVar(&gCmdLineArgs.showMetricNames, "l", false, "") flag.BoolVar(&gCmdLineArgs.showMetricNames, "list", false, "") + flag.StringVar(&gCmdLineArgs.metricsList, "metrics", "", "") + flag.BoolVar(&gCmdLineArgs.printCSV, "csv", false, "") + flag.BoolVar(&gCmdLineArgs.printWide, "wide", false, "") + flag.BoolVar(&gCmdLineArgs.verbose, "v", false, "") + flag.BoolVar(&gCmdLineArgs.veryVerbose, "vv", false, "") flag.IntVar(&gCmdLineArgs.timeout, "t", 0, "") flag.IntVar(&gCmdLineArgs.timeout, "timeout", 0, "") + // process mode options + flag.BoolVar(&gCmdLineArgs.processMode, "per-process", false, "") + flag.StringVar(&gCmdLineArgs.pidList, "p", "", "") + flag.StringVar(&gCmdLineArgs.pidList, "pid", "", "") + flag.StringVar(&gCmdLineArgs.processFilter, "process-filter", "", "") + flag.IntVar(&gCmdLineArgs.processCount, "process-count", 5, "") + flag.IntVar(&gCmdLineArgs.processRefresh, "process-refresh", 30, "") + // advanced options flag.IntVar(&gCmdLineArgs.perfPrintInterval, "i", 5000, "") flag.IntVar(&gCmdLineArgs.perfPrintInterval, "interval", 5000, "") flag.IntVar(&gCmdLineArgs.perfMuxInterval, "x", 125, "") @@ -375,15 +565,20 @@ func mainReturnWithCode() int { flag.StringVar(&gCmdLineArgs.eventFilePath, "eventfile", "", "") flag.StringVar(&gCmdLineArgs.metricFilePath, "m", "", "") flag.StringVar(&gCmdLineArgs.metricFilePath, "metricfile", "", "") - flag.BoolVar(&gCmdLineArgs.printCSV, "csv", false, "") - flag.BoolVar(&gCmdLineArgs.printWide, "wide", false, "") - flag.StringVar(&gCmdLineArgs.metricsList, "metrics", "", "") - flag.BoolVar(&gCmdLineArgs.verbose, "v", false, "") - flag.BoolVar(&gCmdLineArgs.veryVerbose, "vv", false, "") // debugging options - flag.StringVar(&gCmdLineArgs.metadataFilePath, "d", "", "") - flag.StringVar(&gCmdLineArgs.perfStatFilePath, "p", "", "") + flag.StringVar(&gCmdLineArgs.metadataFilePath, "metadata", "", "") + flag.StringVar(&gCmdLineArgs.perfStatFilePath, "perfstat", "", "") flag.Parse() +} + +const ( + exitNoError = 0 + exitError = 1 + exitInterrupt = 2 +) + +func mainReturnWithCode() int { + configureArgs() err := validateArgs() if err != nil { log.Printf("Invalid argument error: %v", err) @@ -470,10 +665,17 @@ func mainReturnWithCode() int { log.Printf("failed to configure metrics: %v", err) return exitError } + var groupDefinitions []GroupDefinition + if groupDefinitions, err = loadEventDefinitions(gCmdLineArgs.eventFilePath, metadata); err != nil { + log.Printf("failed to load event definitions: %v", err) + return exitError + } + if !gCmdLineArgs.printCSV { + fmt.Print(".") + } if gCmdLineArgs.perfStatFilePath != "" { // testing/debugging flow fmt.Print(".\n") - gCollectionStartTime = time.Now() - if err = doWorkDebug(gCmdLineArgs.perfStatFilePath, metricDefinitions, metadata); err != nil { + if err = doWorkDebug(gCmdLineArgs.perfStatFilePath, groupDefinitions, metricDefinitions, metadata); err != nil { log.Printf("%v", err) return exitError } @@ -493,14 +695,6 @@ func mainReturnWithCode() int { if gCmdLineArgs.verbose { log.Printf("Using perf at %s.", perfPath) } - var groupDefinitions []GroupDefinition - if groupDefinitions, err = loadEventDefinitions(gCmdLineArgs.eventFilePath, metadata); err != nil { - log.Printf("failed to load event definitions: %v", err) - return exitError - } - if !gCmdLineArgs.printCSV { - fmt.Print(".") - } var nmiWatchdog string if nmiWatchdog, err = getNmiWatchdog(); err != nil { log.Printf("failed to retrieve NMI watchdog status: %v", err) diff --git a/src/pmu2metrics/metrics.go b/src/pmu2metrics/metrics.go index 9c522b6..e30692f 100644 --- a/src/pmu2metrics/metrics.go +++ b/src/pmu2metrics/metrics.go @@ -5,6 +5,7 @@ import ( "log" "math" "strings" + "sync" "github.com/Knetic/govaluate" mapset "github.com/deckarep/golang-set/v2" @@ -15,8 +16,29 @@ type Metric struct { Value float64 } +// lock to protect metric variable map that holds the event group where a variable value will be retrieved +var metricVariablesLock = sync.RWMutex{} + // for each variable in a metric, set the best group from which to get its value func loadMetricBestGroups(metric MetricDefinition, frame EventFrame) (err error) { + // one thread at a time through this function, since it updates the metric variables map and this only needs to be done one time + metricVariablesLock.Lock() + defer metricVariablesLock.Unlock() + // only load event groups one time for each metric + loadGroups := false + for variableName := range metric.Variables { + if metric.Variables[variableName] == -1 { // group not yet set + loadGroups = true + break + } + if metric.Variables[variableName] == -2 { // tried previously and failed, don't try again + err = fmt.Errorf("metric variable group assignment previously failed, skipping: %s", variableName) + return + } + } + if !loadGroups { + return // nothing to do, already loaded + } allVariableNames := mapset.NewSetFromMapKeys(metric.Variables) remainingVariableNames := allVariableNames.Clone() for { @@ -58,28 +80,9 @@ func loadMetricBestGroups(metric MetricDefinition, frame EventFrame) (err error) // get the variable values that will be used to evaluate the metric's expression func getExpressionVariableValues(metric MetricDefinition, frame EventFrame, previousTimestamp float64, metadata Metadata) (variables map[string]interface{}, err error) { variables = make(map[string]interface{}) - // if first frame, we'll need to determine the best groups from which to get event values for the variables - loadGroups := false - for variableName := range metric.Variables { - if metric.Variables[variableName] == -1 { // group not yet set - loadGroups = true - } - if metric.Variables[variableName] == -2 { // tried previously and failed, don't try again - err = fmt.Errorf("metric variable group assignment previously failed, skipping: %s", variableName) - if gCmdLineArgs.veryVerbose { - log.Print(err.Error()) - } - return - } - } - if loadGroups { - if err = loadMetricBestGroups(metric, frame); err != nil { - err = fmt.Errorf("at least one of the variables couldn't be assigned to a group: %v", err) - if gCmdLineArgs.veryVerbose { - log.Print(err.Error()) - } - return - } + if err = loadMetricBestGroups(metric, frame); err != nil { + err = fmt.Errorf("at least one of the variables couldn't be assigned to a group: %v", err) + return } // set the variable values to be used in the expression evaluation for variableName := range metric.Variables { @@ -150,7 +153,7 @@ func evaluateExpression(metric MetricDefinition, variables map[string]interface{ } }() if result, err = metric.Evaluable.Evaluate(variables); err != nil { - log.Printf("%v : %s : %s", err, metric.Name, metric.Expression) + err = fmt.Errorf("%v : %s : %s", err, metric.Name, metric.Expression) } return } @@ -170,12 +173,14 @@ func processEvents(perfEvents [][]byte, metricDefinitions []MetricDefinition, pr if gCmdLineArgs.verbose { log.Printf("failed to get expression variable values: %v", err) } + err = nil } else { var result interface{} if result, err = evaluateExpression(metricDef, variables); err != nil { if gCmdLineArgs.verbose { log.Printf("failed to evaluate expression: %v", err) } + err = nil } else { metric.Value = result.(float64) } diff --git a/src/pmu2metrics/process.go b/src/pmu2metrics/process.go new file mode 100644 index 0000000..c00f99e --- /dev/null +++ b/src/pmu2metrics/process.go @@ -0,0 +1,101 @@ +package main + +import ( + "bytes" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" +) + +func processExists(pid string) (exists bool) { + exists = false + path := filepath.Join("/", "proc", pid) + if fileInfo, err := os.Stat(path); err == nil { + if fileInfo.Mode().IsDir() { + exists = true + } + } + return +} + +type Process struct { + pid string + ppid string + comm string + cmd string +} + +// match output of, and capture fields from, ps commands used below +var psRegex = `^\s*(\d+)\s+(\d+)\s+([\w\d\(\)\:\/_\-\:\.]+)\s+(.*)` + +func getProcess(pid string) (process Process, err error) { + cmd := exec.Command("ps", "-q", pid, "h", "-o", "pid,ppid,comm,cmd", "ww") + var outBuffer, errBuffer bytes.Buffer + cmd.Stderr = &errBuffer + cmd.Stdout = &outBuffer + if err = cmd.Run(); err != nil { + return + } + psOutput := outBuffer.String() + reProcess := regexp.MustCompile(psRegex) + match := reProcess.FindStringSubmatch(psOutput) + if match == nil { + err = fmt.Errorf("Process not found, PID: %s, ps output: %s", pid, psOutput) + return + } + process = Process{pid: match[1], ppid: match[2], comm: match[3], cmd: match[4]} + return +} + +// get maxProcesses processes with highest CPU utilization, matching filter if provided +func getHotProcesses(maxProcesses int, filter string) (processes []Process, err error) { + // run ps to get list of processes sorted by cpu utilization (descending) + // e.g., ps -e h -o pid,ppid,comm,cmd ww --sort=-%cpu + cmd := exec.Command("ps", "-e", "h", "-o", "pid,ppid,comm,cmd", "ww", "--sort=-%cpu") + var outBuffer, errBuffer bytes.Buffer + cmd.Stderr = &errBuffer + cmd.Stdout = &outBuffer + if err = cmd.Run(); err != nil { + return + } + psOutput := outBuffer.String() + var reFilter *regexp.Regexp + if filter != "" { + if reFilter, err = regexp.Compile(filter); err != nil { + return + } + } + reProcess := regexp.MustCompile(psRegex) + for _, line := range strings.Split(psOutput, "\n") { + match := reProcess.FindStringSubmatch(line) + if match == nil { + log.Printf("Error regex not matching ps output: %s", line) + continue + } + pid := match[1] + ppid := match[2] + comm := match[3] + cmd := match[4] + if !processExists(pid) { + continue + } + if (reFilter != nil && reFilter.MatchString(cmd)) || reFilter == nil { + processes = append(processes, Process{pid: pid, ppid: ppid, comm: comm, cmd: cmd}) + } + if len(processes) == maxProcesses { + break + } + } + if gCmdLineArgs.veryVerbose { + var pids []string + for _, process := range processes { + pids = append(pids, process.pid) + } + log.Printf("Hot PIDs: %s", strings.Join(pids, ", ")) + } + return +} diff --git a/src/reporter/source.go b/src/reporter/source.go index b680c7c..c4257eb 100644 --- a/src/reporter/source.go +++ b/src/reporter/source.go @@ -1125,7 +1125,10 @@ func (s *Source) getPMUMetrics() (orderedMetricNames []string, timeStamps []floa // 1st row has metric names, 2nd-nth have metric values // 1st col has timestamp for colIdx := 0; colIdx < len(rows[0]); colIdx++ { - if rows[0][colIdx] == "" { // skip metrics with no name (can occur in some situations) + label := rows[0][colIdx] + // skip metrics with no name (can occur in some rate situations) + // skip PID and CMD -- not used in system mode + if label == "" || label == "PID" || label == "CMD" { continue } if colIdx != 0 { // don't put timestamp column in the metric names list From 08e723474850646b9d7da8be8288caffc087ac41 Mon Sep 17 00:00:00 2001 From: jharper5 Date: Sun, 17 Dec 2023 09:33:47 -0800 Subject: [PATCH 11/49] change dumpconfig arg to printconfig --- src/orchestrator/command_line_args.go | 10 +++++----- src/orchestrator/main.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/orchestrator/command_line_args.go b/src/orchestrator/command_line_args.go index ed7351b..95bd3ad 100644 --- a/src/orchestrator/command_line_args.go +++ b/src/orchestrator/command_line_args.go @@ -37,7 +37,7 @@ type CmdLineArgs struct { output string targetTemp string temp string - dumpConfig bool + printConfig bool noConfig bool cmdTimeout int reporter string @@ -57,7 +57,7 @@ func showUsage() { fmt.Fprintf(os.Stderr, " [-analyze SELECT] [-analyze_duration SECONDS] [-analyze_frequency N]\n") fmt.Fprintf(os.Stderr, " [-megadata]\n") fmt.Fprintf(os.Stderr, " [-ip IP] [-port PORT] [-user USER] [-key KEY] [-targets TARGETS]\n") - fmt.Fprintf(os.Stderr, " [-output OUTPUT] [-temp TEMP] [-targettemp TEMP] [-dumpconfig] [-cmd_timeout]\n") + fmt.Fprintf(os.Stderr, " [-output OUTPUT] [-temp TEMP] [-targettemp TEMP] [-printconfig] [-noconfig] [-cmd_timeout]\n") fmt.Fprintf(os.Stderr, " [-reporter \"args\"] [-collector \"args\"] [-debug]\n") longHelp := ` @@ -89,7 +89,7 @@ analyze arguments: -analyze_frequency N the number of samples taken per second (default: 11) additional data collection arguments: - -megadata collect additional data specified in megadata template file (default: False) + -megadata collect additional data in megadata directory (default: False) remote target arguments: -ip IP ip address or hostname (default: Nil) @@ -106,7 +106,7 @@ advanced arguments: -output DIR path to output directory. Directory must exist. (default: $PWD/orchestrator_timestamp) -temp DIR path to temporary directory on localhost. Directory must exist. (default: system default) -targettemp DIR path to temporary directory on target. Directory must exist. (default: system default) - -dumpconfig dump the collector configuration file and exit (default: False) + -printconfig print the collector configuration file and exit (default: False) -noconfig do not collect system configuration data. (default: False) -cmd_timeout the maximum number of seconds to wait for each data collection command (default: 300) -reporter run the the reporter sub-component with args @@ -147,7 +147,7 @@ func (cmdLineArgs *CmdLineArgs) parse(name string, arguments []string) (err erro flagSet.StringVar(&cmdLineArgs.output, "output", "", "") flagSet.StringVar(&cmdLineArgs.temp, "temp", "", "") flagSet.StringVar(&cmdLineArgs.targetTemp, "targettemp", "", "") - flagSet.BoolVar(&cmdLineArgs.dumpConfig, "dumpconfig", false, "") + flagSet.BoolVar(&cmdLineArgs.printConfig, "printconfig", false, "") flagSet.BoolVar(&cmdLineArgs.noConfig, "noconfig", false, "") flagSet.IntVar(&cmdLineArgs.cmdTimeout, "cmd_timeout", 300, "") flagSet.StringVar(&cmdLineArgs.format, "format", "html,xlsx,json", "") diff --git a/src/orchestrator/main.go b/src/orchestrator/main.go index 389bf5e..c445d4f 100644 --- a/src/orchestrator/main.go +++ b/src/orchestrator/main.go @@ -290,7 +290,7 @@ func cleanupOutputDir(outputDir string, collections []*Collection, reportFilePat } func (app *App) doWork() (err error) { - if app.args.dumpConfig { + if app.args.printConfig { var bytes []byte bytes, err = resources.ReadFile("resources/collector_reports.yaml.tmpl") if err != nil { From 37e137e1880d3359772095190d6a60a3a7ef926c Mon Sep 17 00:00:00 2001 From: jharper5 Date: Sun, 17 Dec 2023 10:00:24 -0800 Subject: [PATCH 12/49] fix bug(s) when -noconfig option is used --- src/pkg/cpu/cpu.go | 4 ++++ src/reporter/report_tables.go | 22 ++++++++++++++-------- src/reporter/source.go | 4 ++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/pkg/cpu/cpu.go b/src/pkg/cpu/cpu.go index f46a11c..0e8bf5c 100644 --- a/src/pkg/cpu/cpu.go +++ b/src/pkg/cpu/cpu.go @@ -150,6 +150,10 @@ func (c *CPU) getMicroArchitectureExt(family, model, sockets string, capid4 stri } func (c *CPU) getCPUByUarch(uarch string) (cpu CPUInfo, err error) { + if uarch == "" { + err = fmt.Errorf("microarchitecture not provided") + return + } for _, info := range c.cpusInfo { var re *regexp.Regexp re, err = regexp.Compile(info.Architecture) diff --git a/src/reporter/report_tables.go b/src/reporter/report_tables.go index 06ff8f7..c4b3450 100644 --- a/src/reporter/report_tables.go +++ b/src/reporter/report_tables.go @@ -848,16 +848,22 @@ func newISATable(sources []*Source, category TableCategory) (table *Table) { }, } flags := source.valFromRegexSubmatch("lscpu", `^Flags.*:\s*(.*)$`) + cpuid := source.getCommandOutput("cpuid -1") for _, isa := range isas { - cpuSupport := yesIfTrue(source.valFromRegexSubmatch("cpuid -1", isa.CPUID+`\s*= (.+?)$`)) - kernelSupport := "Yes" - match, err := regexp.MatchString(" "+isa.lscpu+" ", flags) - if err != nil { - log.Printf("regex match failed: %v", err) - return + var kernelSupport, cpuSupport string + if cpuid != "" { + cpuSupport = yesIfTrue(source.valFromRegexSubmatch("cpuid -1", isa.CPUID+`\s*= (.+?)$`)) } - if !match { - kernelSupport = "No" + if flags != "" { + kernelSupport = "Yes" + match, err := regexp.MatchString(" "+isa.lscpu+" ", flags) + if err != nil { + log.Printf("regex match failed: %v", err) + return + } + if !match { + kernelSupport = "No" + } } hostValues.Values = append(hostValues.Values, []string{isa.Name, isa.FullName, cpuSupport, kernelSupport}) } diff --git a/src/reporter/source.go b/src/reporter/source.go index c4257eb..7058b40 100644 --- a/src/reporter/source.go +++ b/src/reporter/source.go @@ -613,6 +613,10 @@ func (s *Source) getL3PerCore(uArch string, coresPerSocketStr string, socketsStr } func (s *Source) getPrefetchers(uarch string) (val string) { + if uarch == "" { + // uarch is required + return + } // MSR_PREFETCH_CONTROL // prefetchers are enabled when associated bit is 0 prefetcherDefs := []struct { From 86e2c064332ed79967deb8b2111bd053326e81ce Mon Sep 17 00:00:00 2001 From: jharper5 Date: Mon, 18 Dec 2023 10:34:13 -0800 Subject: [PATCH 13/49] update third party programs copyright/license file --- THIRD_PARTY_PROGRAMS | 456 +++++++++++++++++++++++++------------------ 1 file changed, 262 insertions(+), 194 deletions(-) diff --git a/THIRD_PARTY_PROGRAMS b/THIRD_PARTY_PROGRAMS index eea3ab6..4b4b98f 100644 --- a/THIRD_PARTY_PROGRAMS +++ b/THIRD_PARTY_PROGRAMS @@ -2,43 +2,30 @@ Intel® System Health Inspector Third Party Programs File This file is the "third-party-programs.txt" file specified in the associated Intel end user license agreement for the Intel software you are licensing. Third party programs and their corresponding required notices and/or license terms are listed below. ------------------------------------------------------------- -DMidecode -Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. -------------------------------------------------------------- -stress-ng -Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -------------------------------------------------------------- -sysstat -License: GPLv2+ -Copyright: - * (C) 1998-2022 by Sebastien GODARD (sysstat orange.fr) +async-profiler + * Copyright 2017 Andrei Pangin * - *************************************************************************** - * This program is free software; you can redistribute it and/or modify it * - * under the terms of the GNU General Public License as published by the * - * Free Software Foundation; either version 2 of the License, or (at your * - * option) any later version. * - * * - * This program is distributed in the hope that it will be useful, but * - * WITHOUT ANY WARRANTY; without the implied warranty of MERCHANTABILITY * - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * - * for more details. * - * * - * You should have received a copy of the GNU General Public License along * - * with this program; if not, write to the Free Software Foundation, Inc., * - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA * - *************************************************************************** -------------------------------------------------------------- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ------------------------------------------------------------ cpuid ** Copyright 2003,2004,2005,2006,2010,2011,2012,2013,2014,2015,2016,2017,2018, ** 2020 by Todd Allen. -Linux Kernel (turbostat) -Copyright (c) 2013 Intel Corporation. - * Len Brown +------------------------------------------------------------- +dmidecode +Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. ------------------------------------------------------------- ethtool * Copyright (C) 1998 David S. Miller (davem@dm.cobaltmicro.com) @@ -71,6 +58,57 @@ Copyright (C) 1989, 1991 Free Software Foundation, Inc., Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. ------------------------------------------------------------- +iostat +* iostat: report CPU and I/O statistics + * (C) 1998-2023 by Sebastien GODARD (sysstat orange.fr) + * + *************************************************************************** + * This program is free software; you can redistribute it and/or modify it * + * under the terms of the GNU General Public License as published by the * + * Free Software Foundation; either version 2 of the License, or (at your * + * option) any later version. * + * * + * This program is distributed in the hope that it will be useful, but * + * WITHOUT ANY WARRANTY; without the implied warranty of MERCHANTABILITY * + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * + * for more details. * + * * + * You should have received a copy of the GNU General Public License along * + * with this program; if not, write to the Free Software Foundation, Inc., * + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA * + *************************************************************************** +------------------------------------------------------------- +ipmitool +Copyright (c) 2003 Sun Microsystems, Inc. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +Redistribution of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +Redistribution in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +Neither the name of Sun Microsystems, Inc. or the names of +contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +This software is provided "AS IS," without a warranty of any kind. +ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES, +INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE OR NON-INFRINGEMENT, ARE HEREBY EXCLUDED. +SUN MICROSYSTEMS, INC. ("SUN") AND ITS LICENSORS SHALL NOT BE LIABLE +FOR ANY DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING +OR DISTRIBUTING THIS SOFTWARE OR ITS DERIVATIVES. IN NO EVENT WILL +SUN OR ITS LICENSORS BE LIABLE FOR ANY LOST REVENUE, PROFIT OR DATA, +OR FOR DIRECT, INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL OR +PUNITIVE DAMAGES, HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF +LIABILITY, ARISING OUT OF THE USE OF OR INABILITY TO USE THIS SOFTWARE, +EVEN IF SUN HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. +------------------------------------------------------------- lshw Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA @@ -171,105 +209,92 @@ NO WARRANTY END OF TERMS AND CONDITIONS ------------------------------------------------------------- -sshpass -Copyright – not available/not listed -shpass was written by Shachar Shemesh for Lingnu Open Source Consulting Ltd. - -GNU General Public License v2.0 or later - -GNU GENERAL PUBLIC LICENSE -Version 2, June 1991 - -Copyright (C) 1989, 1991 Free Software Foundation, Inc. -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -Everyone is permitted to copy and distribute verbatim copies -of this license document, but changing it is not allowed. - -Preamble - -The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. - -When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. - -To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. - -For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. - -We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. - -Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. - -Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. - -The precise terms and conditions for copying, distribution and modification follow. - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. - -1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. - -You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: - -a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. - -b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. - -c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. - -3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: - -a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, - -b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, - -c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. - -If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. - -4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. - -5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. - -6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. - -7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. - -This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. +lspci + * Copyright (c) 1997--2020 Martin Mares + * + * Can be freely distributed and used under the terms of the GNU GPL v2+. + * + * SPDX-License-Identifier: GPL-2.0-or-later +------------------------------------------------------------- +mpstat + * (C) 2000-2023 by Sebastien GODARD (sysstat orange.fr) + * Copyright (C) 2022 Oracle and/or its affiliates. + * + *************************************************************************** + * This program is free software; you can redistribute it and/or modify it * + * under the terms of the GNU General Public License as published by the * + * Free Software Foundation; either version 2 of the License, or (at your * + * option) any later version. * + * * + * This program is distributed in the hope that it will be useful, but * + * WITHOUT ANY WARRANTY; without the implied warranty of MERCHANTABILITY * + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * + * for more details. * + * * + * You should have received a copy of the GNU General Public License along * + * with this program; if not, write to the Free Software Foundation, Inc., * + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA * + *************************************************************************** +------------------------------------------------------------- +perf +The Linux Kernel is provided under: -8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. + SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. +Being under the terms of the GNU General Public License version 2 only, +according with: -Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. + LICENSES/preferred/GPL-2.0 -10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. +With an explicit syscall exception, as stated at: -NO WARRANTY + LICENSES/exceptions/Linux-syscall-note -11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. +In addition, other licenses may also apply. Please see: -12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + Documentation/process/license-rules.rst -END OF TERMS AND CONDITIONS +for more details. +All contributions to the Linux Kernel are subject to this COPYING file. +------------------------------------------------------------- +sadc + * (C) 1999-2023 by Sebastien GODARD (sysstat orange.fr) + * + *************************************************************************** + * This program is free software; you can redistribute it and/or modify it * + * under the terms of the GNU General Public License as published by the * + * Free Software Foundation; either version 2 of the License, or (at your * + * option) any later version. * + * * + * This program is distributed in the hope that it will be useful, but * + * WITHOUT ANY WARRANTY; without the implied warranty of MERCHANTABILITY * + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * + * for more details. * + * * + * You should have received a copy of the GNU General Public License along * + * with this program; if not, write to the Free Software Foundation, Inc., * + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA * + *************************************************************************** +------------------------------------------------------------- +sar + * (C) 1999-2023 by Sebastien GODARD (sysstat orange.fr) + * + *************************************************************************** + * This program is free software; you can redistribute it and/or modify it * + * under the terms of the GNU General Public License as published by the * + * Free Software Foundation; either version 2 of the License, or (at your * + * option) any later version. * + * * + * This program is distributed in the hope that it will be useful, but * + * WITHOUT ANY WARRANTY; without the implied warranty of MERCHANTABILITY * + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * + * for more details. * + * * + * You should have received a copy of the GNU General Public License along * + * with this program; if not, write to the Free Software Foundation, Inc., * + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA * + *************************************************************************** ------------------------------------------------------------- spectre-meltdown-checker Copyright not available/not identified @@ -388,113 +413,156 @@ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY C If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS ------------------------------------------------------------- -ipmitool -Copyright (c) 2003 Sun Microsystems, Inc. All Rights Reserved. +sshpass +Copyright – not available/not listed +shpass was written by Shachar Shemesh for Lingnu Open Source Consulting Ltd. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: +GNU General Public License v2.0 or later -Redistribution of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. +GNU GENERAL PUBLIC LICENSE +Version 2, June 1991 -Redistribution in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. +Copyright (C) 1989, 1991 Free Software Foundation, Inc. +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -Neither the name of Sun Microsystems, Inc. or the names of -contributors may be used to endorse or promote products derived -from this software without specific prior written permission. +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. -This software is provided "AS IS," without a warranty of any kind. -ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES, -INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE OR NON-INFRINGEMENT, ARE HEREBY EXCLUDED. -SUN MICROSYSTEMS, INC. ("SUN") AND ITS LICENSORS SHALL NOT BE LIABLE -FOR ANY DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING -OR DISTRIBUTING THIS SOFTWARE OR ITS DERIVATIVES. IN NO EVENT WILL -SUN OR ITS LICENSORS BE LIABLE FOR ANY LOST REVENUE, PROFIT OR DATA, -OR FOR DIRECT, INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL OR -PUNITIVE DAMAGES, HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF -LIABILITY, ARISING OUT OF THE USE OF OR INABILITY TO USE THIS SOFTWARE, -EVEN IF SUN HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. -------------------------------------------------------------- -Intel® Memory Latency Checker -Copyright: Not available/not listed +Preamble -idzla-Software License for Intel® Memory Latency Checker (Intel® MLC) -SOFTWARE TOOLS LICENSE AGREEMENT +The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. -DO NOT DOWNLOAD, INSTALL, ACCESS, COPY, OR USE ANY PORTION OF THE MATERIALS (DEFINED BELOW) UNTIL YOU HAVE READ AND ACCEPTED THE TERMS AND CONDITIONS OF THIS AGREEMENT. BY INSTALLING, COPYING, ACCESSING, OR USING THE MATERIALS, YOU AGREE TO BE LEGALLY BOUND BY THE TERMS AND CONDITIONS OF THIS AGREEMENT. If You do not agree to be bound by, or the entity for whose benefit You act has not authorized You to accept, these terms and conditions, do not install, access, copy, or use the Software and destroy all copies of the Software in Your possession. +When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. -This DEVELOPMENT TOOLS LICENSE AGREEMENT (this “Agreement”) is entered into between Intel Corporation, a Delaware corporation (“Intel”) and You. “You” refers to you or your employer or other entity for whose benefit you act, as applicable. If you are agreeing to the terms and conditions of this Agreement on behalf of a company or other legal entity, you represent and warrant that you have the legal authority to bind that legal entity to the Agreement, in which case, "You" or "Your" shall be in reference to such entity. Intel and You are referred to herein individually as a “Party” or, together, as the “Parties”. +To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. -The Parties, in consideration of the mutual covenants contained in this Agreement, and for other good and valuable consideration, the receipt and sufficiency of which they acknowledge, and intending to be legally bound, agree as follows: +For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. -1. DEFINITIONS. The following definitions are used throughout this Agreement: +We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. -“Affiliate” means any entity controlling, controlled by or under common control with a Party hereto, where “control” means the direct or indirect ownership of more than fifty percent (50%) of such entity’s capital or equivalent voting rights. An entity will be deemed an “Affiliate” only as long as such control exists during the term of this Agreement. +Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. -“Contractor” means a third party consultant or subcontractor who requires access to or use of the Materials to perform work on Your behalf or at Your behest. +Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. -“Development Tools” means the development, evaluation, production, or test tool software, and associated documentation or other collateral, identified in the “development_tools.txt” text files, if any, included in the Materials. +The precise terms and conditions for copying, distribution and modification follow. -“Derivatives” means derivative works as defined in 17 U.S.C § 101 et seq. +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION -“Intel-based Device” means a device designed, manufactured, or configured by You or Your Affiliates to include or operate Intel hardware, software, or services. +0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". -"Materials" means the software, documentation, the software product serial number and license key codes (if applicable), Development Tools, Redistributables, and other materials or collateral, including any updates and upgrades thereto, in source code or object code form where applicable, that are provided or otherwise made available by Intel to You under this Agreement. “Materials” do not include Open Source Software or any computer programming code that is subject to an agreement, obligation or license (whether or not accompanying the Materials) intended to supersede this Agreement. +Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. -"Redistributables" means the software, documentation, or other collateral identified in the “redist.txt” text files, if any, included in the Materials. +1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. -2. LIMITED LICENSE. +You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. -(A) Subject to the terms and conditions of this Agreement, Intel grants You and Your Affiliates, a limited, nonexclusive, nontransferable, revocable, worldwide, fully paid-up license during the term of this Agreement, without the right to sublicense, under Intel’s copyrights (subject to any third party licensing requirements), unless expressly stated otherwise, to: +2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: -(1) internally reproduce and install a reasonable number of copies of the Materials for Your internal use solely for the purposes of designing, developing, manufacturing and testing Intel-based Devices; +a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. + +b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. -(2) internally reproduce the source code of the Development Tools, if provided to You by Intel, and to internally create and reproduce Derivatives of the Development Tools, and to internally reproduce the binary code of the Development Tools, or any Derivatives created by You, in each case solely for the purpose of designing, developing, manufacturing and testing the Intel-based Device, solely as necessary for the integration of any Intel software and the output generated by the Development Tools, with and into Intel-based Devices; +c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) -(3) create Derivatives of the Redistributables, or any portions thereof, provided to You by Intel in source code form solely for the purposes of designing, developing, debugging, modifying, distributing and testing software containing significantly more functionality and features than the Redistributables in the form provided to You by Intel; +These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. -(4) distribute (or otherwise make available) on a royalty-free basis, subject to any other terms and conditions which may appear in the Redistributables text files, the Redistributables, including any Derivatives of the Redistributables pursuant to Section 2(A)(3), or any portions thereof, only as integrated or embedded in software (and not on a stand-alone basis) solely for use on an Intel-based Device; and +Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. -(5) have the tasks set forth in Section 2(A)(1) and (2) above performed by a Contractor on the conditions that You enter into a written confidentiality agreement with any such Contractor, subject to Section 7 (Confidentiality), and You remain fully liable to Intel for the actions and inactions of Your Contractors. +In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. + +3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: + +a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, + +b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, + +c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. + +If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. -(B) You will be liable for Your Affiliate’s breach of these terms. In addition, You acknowledge that Your Affiliates are beneficiaries of the licenses granted by Intel under Section 2. +5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. -(C) Intel hereby grants You the right to sub-license (without rights to further sublicense) the Development Tools, including any accompanying documentation, to Your manufacturing partners, in the code format provided to You by Intel, solely for designing, developing, manufacturing and testing the Intel-based Devices solely as necessary for the integration of any Intel software and the output generated by the Development Tools, with and into Intel-based Devices. The sublicense is subject to a written sublicensing agreement that contains confidentiality obligations and license restrictions that are no less protective of Intel than those provided in this Agreement. You will be fully responsible and liable towards Intel for Your sub-licensees’ compliance with all such confidentiality obligations and license restrictions. You may grant Your manufacturing partners the right to further distribute Redistributables solely as integrated or embedded in software for Your Intel-based Devices. +6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. -3. LICENSE RESTRICTIONS. All right, title and interest in and to the Materials and associated documentation are and will remain the exclusive property of Intel and its suppliers. Unless expressly permitted under the Agreement, You will not, and will not allow any third party to (i) use, copy, distribute, sell or offer to sell the Materials or associated documentation; (ii) modify, adapt, enhance, disassemble, decompile, reverse engineer, change or create derivative works from the Materials except and only to the extent as specifically required by mandatory applicable laws or any applicable third party license terms accompanying the Materials; (iii) use or make the Materials available for the use or benefit of third parties; or (iv) use the Materials on Your products other than those that include the Intel product(s), platform(s), or software identified in the Materials; or (v) publish or provide any Materials benchmark or comparison test results. +7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. -If You received the Materials solely for evaluation purposes, You have no distribution rights to the Materials or any portion thereof. +If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. -Distribution of the Redistributables is also subject to the following conditions: You shall: (i) be solely responsible to Your customers and end users for any update or support obligation or other liability which may arise from the distribution, (ii) not make any statement that Your software is "certified", or that its performance is guaranteed, by Intel, (iii) not use Intel's name or trademarks to promote Your software without prior written permission, (iv) use a license agreement that contains provisions that are at least as restrictive as this Agreement and which prohibits disassembly and reverse engineering of the Materials provided in object code form, and (v) indemnify, hold harmless, and defend Intel, Intel’s Affiliates, and its licensors from and against any claims or lawsuits, including attorney's fees, that arise or result from Your Derivatives or Your distribution of Your software. +It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. -The consideration under this Agreement is only for the licenses Intel expressly grants above. Any other rights including, but not limited to, additional patent rights, will require an additional license and additional consideration. Nothing in this Agreement requires or will be treated to require Intel to grant any additional license. You acknowledge that an essential basis of the bargain in this Agreement is that Intel grants You no licenses or other rights including, but not limited to, patent, copyright, trade secret, trademark, trade name, service mark or other intellectual property licenses or rights with respect to the Materials and associated documentation, by implication, estoppel or otherwise, except for the licenses expressly granted above. You acknowledge there are significant uses of the Materials in their original, unmodified and uncombined form. The consideration for the licenses in this Agreement reflects Intel’s continuing right to assert patent claims against any modifications or derivative works (including, without limitation, error corrections and bug fixes) of, or combinations with, the Materials that You, Your Affiliates or third parties make that infringe any Intel patent claim. +This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. -4. LICENSE TO FEEDBACK. This Agreement does not obligate You to provide Intel with materials, information, comments, suggestions, Your Derivatives or other communication regarding the features, functions, performance or use of the Materials (“Feedback”). If any software included in the Materials is provided or otherwise made available by Intel in source code form, to the extent You provide Intel with Feedback in a tangible form, You grant to Intel and its affiliates a non-exclusive, perpetual, sublicenseable, irrevocable, worldwide, royalty-free, fully paid-up and transferable license, to and under all of Your intellectual property rights, whether perfected or not, to publicly perform, publicly display, reproduce, use, make, have made, sell, offer for sale, distribute, import, create derivative works of and otherwise exploit any comments, suggestions, descriptions, ideas, Your Derivatives or other feedback regarding the Materials provided by You or on Your behalf. +8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. -5. OPEN SOURCE STATEMENT. The Materials may include Open Source Software (OSS) licensed pursuant to OSS license agreement(s) identified in the OSS comments in the applicable source code file(s) and/or file header(s) provided with or otherwise associated with the Materials. Neither You nor any Original Equipment Manufacturer (OEM), Original Device Manufacturer (ODM), customer, or distributor may subject any proprietary portion of the Materials to any OSS license obligations including, without limitation, combining or distributing the Materials with OSS in a manner that subjects Intel, the Materials or any portion thereof to any OSS license obligation. Nothing in this Agreement limits any rights under, or grants rights that supersede, the terms of any applicable OSS license. +9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. -6. THIRD PARTY SOFTWARE. Certain third party software provided with or within the Materials may only be used (a) upon securing a license directly from the owner of the software or (b) in combination with hardware components purchased from such third party and (c) subject to further license limitations by the software owner. A listing of any such third party limitations is in one or more text files accompanying the Materials. You acknowledge Intel is not providing You with a license to such third party software and further that it is Your responsibility to obtain appropriate licenses from such third parties directly. +Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. -7. CONFIDENTIALITY. The terms and conditions of this Agreement, exchanged confidential information, as well as the Materials are subject to the terms and conditions of the Non-Disclosure Agreement(s) or Intel Pre-Release Loan Agreement(s) (referred to herein collectively or individually as “NDA”) entered into by and in force between Intel and You, and in any case no less confidentiality protection than You apply to Your information of similar sensitivity. If You would like to have a Contractor perform work on Your behalf that requires any access to or use of Materials You must obtain a written confidentiality agreement from the Contractor which contains terms and conditions with respect to access to or use of Materials no less restrictive than those set forth in this Agreement, excluding any distribution rights and use for any other purpose, and You will remain fully liable to Intel for the actions and inactions of those Contractors. You may not use Intel's name in any publications, advertisements, or other announcements without Intel's prior written consent. +10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. -8. NO OBLIGATION; NO AGENCY. Intel may make changes to the Software, or items referenced therein, at any time without notice. Intel is not obligated to support, update, provide training for, or develop any further version of the Software or to grant any license thereto. No agency, franchise, partnership, joint- venture, or employee-employer relationship is intended or created by this Agreement. +NO WARRANTY -9. EXCLUSION OF WARRANTIES. THE MATERIALS ARE PROVIDED "AS IS" WITHOUT ANY EXPRESS OR IMPLIED WARRANTY OF ANY KIND INCLUDING WARRANTIES OF MERCHANTABILITY, NONINFRINGEMENT, OR FITNESS FOR A PARTICULAR PURPOSE. Intel does not warrant or assume responsibility for the accuracy or completeness of any information, text, graphics, links or other items within the Materials. +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. -10. LIMITATION OF LIABILITY. IN NO EVENT WILL INTEL OR ITS AFFILIATES, LICENSORS OR SUPPLIERS (INCLUDING THEIR RESPECTIVE DIRECTORS, OFFICERS, EMPLOYEES, AND AGENTS) BE LIABLE FOR ANY DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, LOST PROFITS, BUSINESS INTERRUPTION, OR LOST DATA) ARISING OUT OF OR IN RELATION TO THIS AGREEMENT, INCLUDING THE USE OF OR INABILITY TO USE THE MATERIALS, EVEN IF INTEL HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. SOME JURISDICTIONS PROHIBIT EXCLUSION OR LIMITATION OF LIABILITY FOR IMPLIED WARRANTIES OR CONSEQUENTIAL OR INCIDENTAL DAMAGES, SO THE ABOVE LIMITATION MAY IN PART NOT APPLY TO YOU. YOU MAY ALSO HAVE OTHER LEGAL RIGHTS THAT VARY FROM JURISDICTION TO JURISDICTION. THE MATERIALS LICENSED HEREUNDER ARE NOT DESIGNED OR INTENDED FOR USE IN ANY MEDICAL, LIFE SAVING OR LIFE SUSTAINING SYSTEMS, TRANSPORTATION SYSTEMS, NUCLEAR SYSTEMS, OR FOR ANY OTHER MISSION CRITICAL APPLICATION IN WHICH THE FAILURE OF THE DEVELOPMENT TOOLS COULD LEAD TO PERSONAL INJURY OR DEATH. YOU WILL INDEMNIFY AND HOLD INTEL AND ITS AFFILIATES, LICENSORS AND SUPPLIERS (INCLUDING THEIR RESPECTIVE DIRECTORS, OFFICERS, EMPLOYEES, AND AGENTS) HARMLESS AGAINST ALL CLAIMS, LIABILITIES, LOSSES, COSTS, DAMAGES, AND EXPENSES (INCLUDING REASONABLE ATTORNEY FEES), ARISING OUT OF, DIRECTLY OR INDIRECTLY, THE DISTRIBUTION OF THE MATERIALS AND ANY CLAIM OF PRODUCT LIABILITY, PERSONAL INJURY OR DEATH ASSOCIATED WITH ANY UNINTENDED USE, EVEN IF SUCH CLAIM ALLEGES THAT INTEL OR AN INTEL AFFILIATE, LICENSOR OR SUPPLIER WAS NEGLIGENT REGARDING THE DESIGN OR MANUFACTURE OF THE MATERIALS. THE LIMITED REMEDIES, WARRANTY DISCLAIMER AND LIMITED LIABILITY ARE FUNDAMENTAL ELEMENTS OF THE BASIS OF THE BARGAIN BETWEEN INTEL AND YOU AND INTEL WOULD NOT BE ABLE TO PROVIDE THE MATERIALS WITHOUT SUCH LIMITATIONS. +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. -11. TERMINATION AND SURVIVAL. Intel may terminate this Agreement for any reason with thirty (30) days’ notice and immediately if You or someone acting on Your behalf or at Your behest violates any of its terms or conditions. Upon termination You will immediately destroy and ensure the destruction of the Materials (including providing certification of such destruction or return back to Intel). Upon termination of this Agreement, all licenses granted to You hereunder terminate immediately. All Sections of this Agreement, except Section 2, will survive termination. In the event of termination of this Agreement, the license grant to any Redistributables, including Your Derivatives of the Redistributables, distributed by You prior to the effective date of such termination and in accordance with the terms and conditions of this Agreement shall survive any such termination of this Agreement. -12. GOVERNING LAW AND JURISDICTION. This Agreement and any dispute arising out of or relating to it will be governed by the laws of the U.S.A. and Delaware, without regard to conflict of laws principles. The Parties exclude the application of the United Nations Convention on Contracts for the International Sale of Goods (1980). The state and federal courts sitting in Delaware, U.S.A. will have exclusive jurisdiction over any dispute arising out of or relating to this Agreement. The Parties consent to personal jurisdiction and venue in those courts. A Party that obtains a judgment against the other Party in the courts identified in this section may enforce that judgment in any court that has jurisdiction over the Parties. -13. EXPORT REGULATIONS/EXPORT CONTROL. You agree that neither You nor Your subsidiaries or Affiliates will export/re-export the Materials, directly or indirectly, to any country for which the U.S. Department of Commerce or any other agency or department of the U.S. Government or the foreign government from where it is shipping requires an export license, or other governmental approval, without first obtaining any such required license or approval. In the event the Materials are exported from the U.S.A. or re-exported from a foreign destination by You, Your subsidiaries, or Your Affiliates, You will ensure that the distribution and export/re-export or import of the Materials complies with all laws, regulations, orders, or other restrictions of the U.S. Export Administration Regulations and the appropriate foreign government. -14. GOVERNMENT RESTRICTED RIGHTS. The Materials are a commercial item (as defined in 48 C.F.R. 2.101) consisting of commercial computer software and commercial computer software documentation (as those terms are used in 48 C.F.R. 12.212). Consistent with 48 C.F.R. 12.212 and 48 C.F.R 227.7202- 1 through 227.7202-4, You will not provide the Materials to the U.S. Government. Contractor or Manufacturer is Intel Corporation, 2200 Mission College Blvd., Santa Clara, CA 95054. -15. TRADEMARKS. Third party trademarks, trade names, product names and logos (the “Trademarks”) contained in or used by the Materials are the trademarks or registered trademarks of their respective owners, and the use of such Trademarks shall inure to the benefit of the trademark owner. The reference to such Trademarks (if any) by Intel in any of the Materials does not constitute: (i) an affiliation by Intel and its licensors with such company, or (ii) an endorsement or approval of such company of Intel and its licensors and its products or services. -16. ASSIGNMENT. You may not delegate, assign or transfer this Agreement, the license(s) granted or any of Your rights or duties hereunder, expressly, by implication, by operation of law, or otherwise and any attempt to do so, without Intel’s express prior written consent, will be null and void. Intel may assign, delegate and transfer this Agreement, and its rights and obligations hereunder, in its sole discretion. -17. ENTIRE AGREEMENT; SEVERABILITY. The terms and conditions of this Agreement and any NDA with Intel constitute the entire agreement between the Parties with respect to the subject matter hereof, and merge and supersede all prior or contemporaneous agreements, understandings, negotiations and discussions. Neither Party will be bound by any terms, conditions, definitions, warranties, understandings, or representations with respect to the subject matter hereof other than as expressly provided herein. In the event any provision of this Agreement is unenforceable or invalid under any applicable law or applicable court decision, such unenforceability or invalidity will not render this Agreement unenforceable or invalid as a whole, instead such provision will be changed and interpreted so as to best accomplish the objectives of such provision within legal limits. -18. WAIVER. The failure of a Party to require performance by the other Party of any provision hereof will not affect the full right to require such performance at any time thereafter; nor will waiver by a Party of a breach of any provision hereof constitute a waiver of the provision itself. -19. PRIVACY. YOUR PRIVACY RIGHTS ARE SET FORTH IN INTEL’S PRIVACY NOTICE, WHICH FORMS A PART OF THIS AGREEMENT. PLEASE REVIEW THE PRIVACY NOTICE AT HTTP://WWW.INTEL.COM/PRIVACY TO LEARN HOW INTEL COLLECTS, USES AND SHARES INFORMATION ABOUT YOU. +END OF TERMS AND CONDITIONS ------------------------------------------------------------- -Other names and brands may be claimed as the property of others. +stackcollapse-perf.pl +# Copyright 2012 Joyent, Inc. All rights reserved. +# Copyright 2012 Brendan Gregg. All rights reserved. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +------------------------------------------------------------- +stress-ng +Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +------------------------------------------------------------- +sysstat +License: GPLv2+ +Copyright: + * (C) 1998-2022 by Sebastien GODARD (sysstat orange.fr) + * + *************************************************************************** + * This program is free software; you can redistribute it and/or modify it * + * under the terms of the GNU General Public License as published by the * + * Free Software Foundation; either version 2 of the License, or (at your * + * option) any later version. * + * * + * This program is distributed in the hope that it will be useful, but * + * WITHOUT ANY WARRANTY; without the implied warranty of MERCHANTABILITY * + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * + * for more details. * + * * + * You should have received a copy of the GNU General Public License along * + * with this program; if not, write to the Free Software Foundation, Inc., * + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA * + *************************************************************************** +------------------------------------------------------------- +turbostat +Copyright (c) 2023 Intel Corporation. + * Len Brown +------------------------------------------------------------- + +Other names and brands may be claimed as the property of others. \ No newline at end of file From 66078462c7dfcbab1d19d5f18c2f95f995c53d3f Mon Sep 17 00:00:00 2001 From: jharper5 Date: Mon, 18 Dec 2023 10:42:33 -0800 Subject: [PATCH 14/49] fix to unit test required due to arg change from dumpconfig to printconfig --- src/orchestrator/command_line_args_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/orchestrator/command_line_args_test.go b/src/orchestrator/command_line_args_test.go index ad5d386..fda4223 100644 --- a/src/orchestrator/command_line_args_test.go +++ b/src/orchestrator/command_line_args_test.go @@ -86,7 +86,7 @@ func TestAllExceptTargetsFile(t *testing.T) { "-megadata", "-debug", "-cmd_timeout", "150", - "-dumpconfig", + "-printconfig", "-ip", "192.168.1.1", "-port", "20", "-user", "foo", From d0b8a5c78f08617411508183aa9aa5925c7dc2e5 Mon Sep 17 00:00:00 2001 From: jharper5 Date: Mon, 18 Dec 2023 10:43:11 -0800 Subject: [PATCH 15/49] add tools directory to svr-info dist tarball --- Makefile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 91f5650..f527690 100644 --- a/Makefile +++ b/Makefile @@ -20,14 +20,13 @@ dist-amd64: tools cp THIRD_PARTY_PROGRAMS dist/svr-info cp docs/guide/SvrInfoUserGuide.pdf dist/svr-info/USER_GUIDE.pdf cp src/orchestrator/orchestrator dist/svr-info/svr-info + mkdir -p dist/svr-info/tools + cp src/orchestrator/resources/* dist/svr-info/tools + cp src/pmu2metrics/pmu2metrics-with-perf dist/svr-info/tools + cd dist/svr-info/tools && tar -xf collector_deps_amd64.tgz && rm collector_deps_*.tgz cd dist && tar -czf $(TARBALL) svr-info cd dist && md5sum $(TARBALL) > $(TARBALL).md5 rm -rf dist/svr-info - rm -rf dist/tools - mkdir -p dist/tools - cp src/orchestrator/resources/* dist/tools - cp src/pmu2metrics/pmu2metrics-with-perf dist/tools - cd dist/tools && tar -xf collector_deps_amd64.tgz && rm collector_deps_*.tgz *.yaml.tmpl dist: dist-amd64 From bcbecf715d6cab282c29c1bf330d606f6fbd7b3a Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Wed, 20 Dec 2023 08:55:00 -0800 Subject: [PATCH 16/49] Fix microarch identification on AWS m7i. (#232) --- src/pkg/cpu/cpu.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pkg/cpu/cpu.go b/src/pkg/cpu/cpu.go index 0e8bf5c..cc1d814 100644 --- a/src/pkg/cpu/cpu.go +++ b/src/pkg/cpu/cpu.go @@ -100,8 +100,9 @@ func (c *CPU) getMicroArchitectureExt(family, model, sockets string, capid4 stri err = fmt.Errorf("no extended architecture info for %s:%s", family, model) return } - var capid4Int, bits int64 - if model == "143" || model == "207" { // SPR and EMR + var bits int64 + if (model == "143" || model == "207") && capid4 != "" { // SPR and EMR + var capid4Int int64 capid4Int, err = strconv.ParseInt(capid4, 16, 64) if err != nil { return From 2f6b0b4a2198f8e422ba047acafd9d6eb2d6612e Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Wed, 3 Jan 2024 12:38:30 -0800 Subject: [PATCH 17/49] remove 'unknown' from uarch names (#235) --- src/pkg/cpu/cpu.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkg/cpu/cpu.go b/src/pkg/cpu/cpu.go index cc1d814..1a27951 100644 --- a/src/pkg/cpu/cpu.go +++ b/src/pkg/cpu/cpu.go @@ -115,7 +115,7 @@ func (c *CPU) getMicroArchitectureExt(family, model, sockets string, capid4 stri } else if bits == 1 { uarch = "SPR_MCC" } else { - uarch = "SPR_Unknown" + uarch = "SPR" } } else if model == "207" { // EMR if bits == 3 { @@ -123,7 +123,7 @@ func (c *CPU) getMicroArchitectureExt(family, model, sockets string, capid4 stri } else if bits == 1 { uarch = "EMR_MCC" } else { - uarch = "EMR_Unknown" + uarch = "EMR" } } else if model == "173" { // GNR var devCount int @@ -144,7 +144,7 @@ func (c *CPU) getMicroArchitectureExt(family, model, sockets string, capid4 stri } else if ratio == 5 { uarch = "GNR_X3" // 3 dies, GNR-AP UCC } else { - uarch = "GNR_Unknown" + uarch = "GNR" } } return From 055dbd46cd740654edab9e8860542d3b71c7132a Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Wed, 3 Jan 2024 12:39:08 -0800 Subject: [PATCH 18/49] update insights (#237) --- src/reporter/resources/insights.grl | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/reporter/resources/insights.grl b/src/reporter/resources/insights.grl index 436dd25..2c457f3 100644 --- a/src/reporter/resources/insights.grl +++ b/src/reporter/resources/insights.grl @@ -21,8 +21,8 @@ rule DIMMSpeed { ( (Report.GetValue("Configuration", "CPU", "Microarchitecture") == "CLX" && Report.GetValueAsInt("Configuration", "DIMM", "Speed") < 2933) || (Report.GetValue("Configuration", "CPU", "Microarchitecture") == "ICX" && Report.GetValueAsInt("Configuration", "DIMM", "Speed") < 3200) || - (Report.GetValue("Configuration", "CPU", "Microarchitecture") == "SPR" && Report.GetValueAsInt("Configuration", "DIMM", "Speed") < 4800) || - (Report.GetValue("Configuration", "CPU", "Microarchitecture") == "EMR" && Report.GetValueAsInt("Configuration", "DIMM", "Speed") < 4800) + (Report.GetValue("Configuration", "CPU", "Microarchitecture").Contains("SPR") && Report.GetValueAsInt("Configuration", "DIMM", "Speed") < 4800) || + (Report.GetValue("Configuration", "CPU", "Microarchitecture").Contains("EMR") && Report.GetValueAsInt("Configuration", "DIMM", "Speed") < 5600) ) then Report.AddInsight( @@ -120,18 +120,6 @@ rule FrequencyGovernor { Retract("FrequencyGovernor"); } -rule IRQBalance { - when - Report.GetValue("Configuration", "NIC", "IRQBalance") != "" && - Report.GetValue("Configuration", "NIC", "IRQBalance") == "Enabled" - then - Report.AddInsight( - "System is using the IRQ Balance service to manage IRQ CPU affinity.", - "Consider manually configuring IRQ CPU affinity for network intensive workloads." - ); - Retract("IRQBalance"); -} - rule TurboBoost { when Report.GetValue("Configuration", "CPU", "Intel Turbo Boost") != "" && From e55ecd22a134a19f344809bbf30181ee92d4aabe Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Wed, 3 Jan 2024 12:40:29 -0800 Subject: [PATCH 19/49] add all-core turbo temperature in benchmark summary (#233) * add all-core turbo temperature in benchmark summary * update reference data to match field name changes --- .../resources/collector_reports.yaml.tmpl | 2 +- src/reporter/report_tables.go | 22 +++++----- src/reporter/resources/reference.yaml | 42 +++++++++---------- src/reporter/source.go | 21 ++++++++-- 4 files changed, 52 insertions(+), 35 deletions(-) diff --git a/src/orchestrator/resources/collector_reports.yaml.tmpl b/src/orchestrator/resources/collector_reports.yaml.tmpl index a3265b4..d7d946e 100644 --- a/src/orchestrator/resources/collector_reports.yaml.tmpl +++ b/src/orchestrator/resources/collector_reports.yaml.tmpl @@ -450,7 +450,7 @@ commands: - label: CPU Turbo Test command: |- # measure tdp and all-core turbo frequency - ((turbostat -i 2 2>/dev/null &) ; stress-ng --cpu 1 -t 20s 2>&1 ; stress-ng --cpu 0 -t 20s 2>&1 ; pkill -9 -f turbostat) | awk '$0~"stress" {print $0} $1=="Package" || $1=="CPU" || $1=="Core" || $1=="Node" {if(f!=1) print $0;f=1} $1=="-" {print $0}' + ((turbostat -i 2 2>/dev/null &) ; stress-ng --cpu 1 -t 20s 2>&1 ; stress-ng --cpu 0 -t 60s 2>&1 ; pkill -9 -f turbostat) | awk '$0~"stress" {print $0} $1=="Package" || $1=="CPU" || $1=="Core" || $1=="Node" {if(f!=1) print $0;f=1} $1=="-" {print $0}' superuser: true modprobe: msr - label: CPU Idle diff --git a/src/reporter/report_tables.go b/src/reporter/report_tables.go index c4b3450..ef0b3cc 100644 --- a/src/reporter/report_tables.go +++ b/src/reporter/report_tables.go @@ -1366,26 +1366,28 @@ func newBenchmarkSummaryTable(sources []*Source, tableMemBandwidthLatency *Table AllHostValues: []HostValues{}, } for _, source := range sources { - singleCoreTurbo, allCoreTurbo, turboTDP := source.getTurbo() + singleCoreTurbo, allCoreTurbo, turboPower, turboTemperature := source.getTurbo() var hostValues = HostValues{ Name: source.getHostname(), ValueNames: []string{ "CPU Speed", - "Single-core Turbo", - "All-core Turbo", - "Turbo TDP", - "Idle TDP", + "Single-core Turbo Frequency", + "All-core Turbo Frequency", + "All-core Turbo Power", + "All-core Turbo Temperature", + "Idle Power", "Memory Peak Bandwidth", "Memory Minimum Latency", "Disk Speed", }, Values: [][]string{ { - source.getCPUSpeed(), // CPU speed - singleCoreTurbo, // single-core turbo - allCoreTurbo, // all-core turbo - turboTDP, // turbo TDP - source.getIdleTDP(), // idle TDP + source.getCPUSpeed(), // CPU speed + singleCoreTurbo, // single-core turbo + allCoreTurbo, // all-core turbo + turboPower, // all-core turbo power + turboTemperature, // all-core turbo temperature + source.getIdlePower(), // idle power source.getPeakBandwidth(tableMemBandwidthLatency), // peak memory bandwidth source.getMinLatency(tableMemBandwidthLatency), // minimum memory latency source.getDiskSpeed(), // disk speed diff --git a/src/reporter/resources/reference.yaml b/src/reporter/resources/reference.yaml index fe156b6..6256383 100644 --- a/src/reporter/resources/reference.yaml +++ b/src/reporter/resources/reference.yaml @@ -7,9 +7,9 @@ BDX_2: Name: Reference (Intel 2S Xeon E5-2699A v4) Summary: CPU Speed: "403415 ops/s" - Single-core Turbo: "3509 MHz" - All-core Turbo: "2980 MHz" - Turbo TDP: "289.90 Watts" + Single-core Turbo Frequency: "3509 MHz" + All-core Turbo Frequency: "2980 MHz" + All-core Turbo Power: "289.90 Watts" Memory Peak Bandwidth: "138.1 GB/s" Memory Minimum Latency: "78 ns" Memory NUMA Bandwidth: @@ -62,9 +62,9 @@ SKX_2: Name: Reference (Intel 2S Xeon Platinum 8180) Summary: CPU Speed: "585157 ops/s" - Single-core Turbo: "3758 MHz" - All-core Turbo: "3107 MHz" - Turbo TDP: "429.07 Watts" + Single-core Turbo Frequency: "3758 MHz" + All-core Turbo Frequency: "3107 MHz" + All-core Turbo Power: "429.07 Watts" Memory Peak Bandwidth: "225.1 GB/s" Memory Minimum Latency: "71 ns" Memory NUMA Bandwidth: @@ -117,9 +117,9 @@ CLX_2: Name: Reference (Intel 2S Xeon Platinum 8280) Summary: CPU Speed: "548644 ops/s" - Single-core Turbo: "3928 MHz" - All-core Turbo: "3296 MHz" - Turbo TDP: "415.93 Watts" + Single-core Turbo Frequency: "3928 MHz" + All-core Turbo Frequency: "3296 MHz" + All-core Turbo Power: "415.93 Watts" Memory Peak Bandwidth: "223.9 GB/s" Memory Minimum Latency: "72 ns" Memory NUMA Bandwidth: @@ -172,10 +172,10 @@ ICX_2: Name: Reference (Intel 2S Xeon Platinum 8380) Summary: CPU Speed: "933644 ops/s" - Single-core Turbo: "3334 MHz" - All-core Turbo: "2950 MHz" - Turbo TDP: "552.00 Watts" - Idle TDP: "175.38 Watts" + Single-core Turbo Frequency: "3334 MHz" + All-core Turbo Frequency: "2950 MHz" + All-core Turbo Power: "552.00 Watts" + Idle Power: "175.38 Watts" Memory Peak Bandwidth: "350.7 GB/s" Memory Minimum Latency: "70 ns" Memory NUMA Bandwidth: @@ -228,10 +228,10 @@ SPR_XCC_2: Name: Reference (Intel 2S Xeon Platinum 8480+) Summary: CPU Speed: "1678712 ops/s" - Single-core Turbo: "3776 MHz" - All-core Turbo: "2996 MHz" - Turbo TDP: "698.35 Watts" - Idle TDP: "349.21 Watts" + Single-core Turbo Frequency: "3776 MHz" + All-core Turbo Frequency: "2996 MHz" + All-core Turbo Power: "698.35 Watts" + Idle Power: "349.21 Watts" Memory Peak Bandwidth: "524.6 GB/s" Memory Minimum Latency: "111.8 ns" Memory NUMA Bandwidth: @@ -283,13 +283,13 @@ SPR_XCC_1: Hostref: Name: Reference (Intel 1S Xeon Platinum 8480+) Summary: - All-core Turbo: "2999 MHz" + All-core Turbo Frequency: "2999 MHz" CPU Speed: "845743 ops/s" - Idle TDP: "163.79 Watts" + Idle Power: "163.79 Watts" Memory Minimum Latency: "112.2 ns" Memory Peak Bandwidth: "264.0 GB/s" - Single-core Turbo: "3783 MHz" - Turbo TDP: "334.68 Watts" + Single-core Turbo Frequency: "3783 MHz" + All-core Turbo Power: "334.68 Watts" Memory NUMA Bandwidth: - - 263646.3 # 0, 0 Memory Bandwidth and Latency: diff --git a/src/reporter/source.go b/src/reporter/source.go index 7058b40..a0bebae 100644 --- a/src/reporter/source.go +++ b/src/reporter/source.go @@ -806,14 +806,17 @@ func (s *Source) getCPUSpeed() (val string) { return } -func (s *Source) getTurbo() (singleCoreTurbo, allCoreTurbo, turboTDP string) { +func (s *Source) getTurbo() (singleCoreTurbo, allCoreTurbo, turboPower, turboTemperature string) { var allTurbos []string var allTDPs []string + var allTemps []string var turbos []string var tdps []string + var temps []string var headers []string idxTurbo := -1 idxTdp := -1 + idxTemp := -1 re := regexp.MustCompile(`\s+`) // whitespace for _, line := range s.getCommandOutputLines("CPU Turbo Test") { if strings.Contains(line, "stress-ng") { @@ -826,6 +829,10 @@ func (s *Source) getTurbo() (singleCoreTurbo, allCoreTurbo, turboTDP string) { tdps = append(tdps, allTDPs[len(allTDPs)-2]) allTDPs = nil } + if idxTemp >= 0 && len(allTemps) >= 2 { + temps = append(temps, allTemps[len(allTemps)-2]) + allTemps = nil + } } continue } @@ -836,6 +843,8 @@ func (s *Source) getTurbo() (singleCoreTurbo, allCoreTurbo, turboTDP string) { idxTurbo = i } else if h == "PkgWatt" { idxTdp = i + } else if h == "PkgTmp" { + idxTemp = i } } continue @@ -847,18 +856,24 @@ func (s *Source) getTurbo() (singleCoreTurbo, allCoreTurbo, turboTDP string) { if idxTdp >= 0 { allTDPs = append(allTDPs, tokens[idxTdp]) } + if idxTemp >= 0 { + allTemps = append(allTemps, tokens[idxTemp]) + } } if len(turbos) == 2 { singleCoreTurbo = turbos[0] + " MHz" allCoreTurbo = turbos[1] + " MHz" } if len(tdps) == 2 { - turboTDP = tdps[1] + " Watts" + turboPower = tdps[1] + " Watts" + } + if len(temps) == 2 { + turboTemperature = temps[1] + " C" } return } -func (s *Source) getIdleTDP() (val string) { +func (s *Source) getIdlePower() (val string) { cmdout := s.getCommandOutputLine("CPU Idle") if cmdout != "" && cmdout != "0.00" { val = cmdout + " Watts" From 5662abdcf5cbe8053a29549b641444399b9fe5d9 Mon Sep 17 00:00:00 2001 From: Jason Harper Date: Wed, 3 Jan 2024 12:49:56 -0800 Subject: [PATCH 20/49] pmu2metrics: CSV and HTML post-processing (#234) --- src/pmu2metrics/main.go | 52 ++- src/pmu2metrics/post_process.go | 321 +++++++++++++ src/pmu2metrics/resources/base.html | 678 ++++++++++++++++++++++++++++ 3 files changed, 1046 insertions(+), 5 deletions(-) create mode 100644 src/pmu2metrics/post_process.go create mode 100644 src/pmu2metrics/resources/base.html diff --git a/src/pmu2metrics/main.go b/src/pmu2metrics/main.go index 0be8978..dd5ead0 100644 --- a/src/pmu2metrics/main.go +++ b/src/pmu2metrics/main.go @@ -25,7 +25,10 @@ type CmdLineArgs struct { pidList string processFilter string processCount int - processRefresh int // secons + processRefresh int // seconds + // post-processing options + inputCSVFilePath string + postProcessingFormat string // advanced options eventFilePath string metricFilePath string @@ -472,7 +475,7 @@ Collection Options: --per-process Enable process mode. Associates metrics with processes. -p, --pid - Comma separated list of pids. Only valid when in process mode (default: None). + Comma separated list of process ids. Only valid when in process mode (default: None). --process-filter Regular expression used to match process names. Valid only when in process mode and --pid not specified (default: None). --process-count @@ -484,12 +487,20 @@ Output Options: --metrics Metric names to include in output. (Quoted and comma separated list.) --csv - CSV formatted output. Best for parsing. + CSV formatted output. Best for parsing. Required for HTML report generation. --wide Wide formatted output. Best used when a small number of metrics are printed. -v[v] Enable verbose, or very verbose (-vv) logging. +Post-processing Options: + --post-process + Path to csv file created from --csv output. + --format