From e4f0bdc7b62459ff50605ea867898ce1026e5905 Mon Sep 17 00:00:00 2001 From: Krishna Dhulipala Date: Thu, 19 Sep 2024 07:58:37 -0700 Subject: [PATCH] ipmitool SEL logging of AER CEs on OpenBMC platforms Signed-off-by: Krishna Dhulipala --- Makefile.am | 6 ++-- configure.ac | 11 ++++++ ras-aer-handler.c | 24 +++++++++++++ ras-aer-handler.h | 1 + ras-events.c | 3 +- ras-events.h | 3 +- rasdaemon.c | 11 +++++- unified-sel.c | 89 +++++++++++++++++++++++++++++++++++++++++++++++ unified-sel.h | 17 +++++++++ 9 files changed, 160 insertions(+), 5 deletions(-) create mode 100644 unified-sel.c create mode 100644 unified-sel.h --- a/Makefile.am +++ b/Makefile.am @@ -76,7 +76,9 @@ endif if WITH_CPU_FAULT_ISOLATION rasdaemon_SOURCES += ras-cpu-isolation.c queue.c endif - +if WITH_OPENBMC_UNIFIED_SEL + rasdaemon_SOURCES += unified-sel.c +endif if WITH_CXL rasdaemon_SOURCES += ras-cxl-handler.c endif @@ -96,7 +98,7 @@ include_HEADERS = config.h ras-events.h ras-devlink-handler.h ras-diskerror-handler.h rbtree.h ras-page-isolation.h \ non-standard-hisilicon.h non-standard-ampere.h ras-memory-failure-handler.h \ ras-cxl-handler.h ras-cpu-isolation.h queue.h non-standard-yitian.h \ - non-standard-jaguarmicro.h trigger.h + non-standard-jaguarmicro.h trigger.h unified-sel.h # This rule can't be called with more than one Makefile job (like make -j8) # I can't figure out a way to fix that --- a/configure.ac +++ b/configure.ac @@ -190,6 +190,16 @@ AS_IF([test "x$enable_amp_ns_decode" = " AM_CONDITIONAL([WITH_AMP_NS_DECODE], [test x$enable_amp_ns_decode = xyes || test x$enable_all = xyes]) AM_COND_IF([WITH_AMP_NS_DECODE], [USE_AMP_NS_DECODE="yes"], [USE_AMP_NS_DECODE="no"]) +AC_ARG_ENABLE([openbmc_unified_sel], + AS_HELP_STRING([--enable-openbmc-unified-sel], [enable OPENBMC_UNIFIED_SEL events (currently experimental)])) + +AS_IF([test "x$enable_openbmc_unified_sel" = "xyes" || test "x$enable_all" = "xyes"], [ + AC_DEFINE(HAVE_OPENBMC_UNIFIED_SEL,1,"have OpenBMC unified SEL") + AC_SUBST([WITH_OPENBMC_UNIFIED_SEL]) +]) +AM_CONDITIONAL([WITH_OPENBMC_UNIFIED_SEL], [test x$enable_openbmc_unified_sel = xyes || test x$enable_all = xyes]) +AM_COND_IF([WITH_OPENBMC_UNIFIED_SEL], [USE_OPENBMC_UNIFIED_SEL="yes"], [USE_OPENBMC_UNIFIED_SEL="no"]) + AC_ARG_ENABLE([jaguar_ns_decode], AS_HELP_STRING([--enable-jaguar-ns-decode], [enable JAGUAR_NS_DECODE events (currently experimental)])) @@ -261,6 +271,7 @@ compile time options summary CXL events : $USE_CXL Memory CE PFA : $USE_MEMORY_CE_PFA AMP RAS errors : $USE_AMP_NS_DECODE + OpenBMC unified : $USE_OPENBMC_UNIFIED_SEL CPU fault isolation : $USE_CPU_FAULT_ISOLATION YITIAN RAS errors : $USE_YITIAN_NS_DECODE JAGUAR RAS errors : $USE_JAGUAR_NS_DECODE --- a/ras-aer-handler.c +++ b/ras-aer-handler.c @@ -25,6 +25,7 @@ #include "ras-logger.h" #include "bitfield.h" #include "ras-report.h" +#include "unified-sel.h" /* bit field meaning for correctable error */ static const char *aer_cor_errors[32] = { @@ -36,12 +37,14 @@ static const char *aer_cor_errors[32] = [12] = "Replay Timer Timeout", [13] = "Advisory Non-Fatal", [14] = "Corrected Internal Error", + [15] = "Header Log Overflow", }; /* bit field meaning for uncorrectable error */ static const char *aer_uncor_errors[32] = { /* Uncorrectable errors */ [4] = "Data Link Protocol", + [5] = "Surprise Link Down", [12] = "Poisoned TLP", [13] = "Flow Control Protocol", [14] = "Completion Timeout", @@ -51,8 +54,23 @@ static const char *aer_uncor_errors[32] [18] = "Malformed TLP", [19] = "ECRC", [20] = "Unsupported Request", + [21] = "ACS Violation", + [22] = "Uncorrected Internal", + [23] = "MC Blocked TLP", + [24] = "AtomicOp Egress Blocked", + [25] = "TLP Prefix Blocked", + [26] = "Poisoned TLP Egrees Blocked", }; +static bool use_ipmitool = false; + +void ras_aer_handler_init(int enable_ipmitool) +{ +#ifdef HAVE_OPENBMC_UNIFIED_SEL + use_ipmitool = (enable_ipmitool > 0) ? 1 : 0; +#endif +} + #define BUF_LEN 1024 int ras_aer_event_handler(struct trace_seq *s, @@ -195,5 +213,11 @@ int ras_aer_event_handler(struct trace_s log(SYSLOG, LOG_WARNING, "Failed to execute ipmitool\n"); #endif +#ifdef HAVE_OPENBMC_UNIFIED_SEL + if (use_ipmitool) + if (openbmc_unified_sel_log(severity_val, ev.dev_name, status_val) < 0) + return -1; +#endif + return 0; } --- a/ras-aer-handler.h +++ b/ras-aer-handler.h @@ -26,4 +26,5 @@ int ras_aer_event_handler(struct trace_s struct tep_record *record, struct tep_event *event, void *context); +void ras_aer_handler_init(int enable_ipmitool); #endif --- a/ras-events.c +++ b/ras-events.c @@ -894,7 +894,7 @@ static int add_event_handler(struct ras_ return 0; } -int handle_ras_events(int record_events) +int handle_ras_events(int record_events, int enable_ipmitool) { int rc, page_size, i; int num_events = 0; @@ -951,6 +951,7 @@ int handle_ras_events(int record_events) "ras", "mc_event"); #ifdef HAVE_AER + ras_aer_handler_init(enable_ipmitool); rc = add_event_handler(ras, pevent, page_size, "ras", "aer_event", ras_aer_event_handler, NULL, AER_EVENT); if (!rc) --- a/ras-events.h +++ b/ras-events.h @@ -109,7 +109,8 @@ enum ghes_severity { /* Function prototypes */ int toggle_ras_mc_event(int enable); +int handle_ras_events(int record_events, int enable_ipmitool); int ras_offline_mce_event(struct ras_mc_offline_event *event); -int handle_ras_events(int record_events); +int handle_ras_events(int record_events, int enable_ipmitool); #endif --- a/rasdaemon.c +++ b/rasdaemon.c @@ -42,6 +42,7 @@ const char *argp_program_bug_address = " struct arguments { int record_events; int enable_ras; + int enable_ipmitool; int foreground; int offline; }; @@ -74,6 +75,11 @@ static error_t parse_opt(int k, char *ar args->record_events++; break; #endif +#ifdef HAVE_OPENBMC_UNIFIED_SEL + case 'i': + args->enable_ipmitool++; + break; +#endif case 'f': args->foreground++; break; @@ -164,6 +170,9 @@ int main(int argc, char *argv[]) {"record", 'r', 0, 0, "record events via sqlite3", 0}, #endif {"foreground", 'f', 0, 0, "run foreground, not daemonize"}, +#ifdef HAVE_OPENBMC_UNIFIED_SEL + {"ipmitool", 'i', 0, 0, "enable ipmitool logging", 0}, +#endif #ifdef HAVE_MCE {"post-processing", 'p', 0, 0, "Post-processing MCE's with raw register values"}, @@ -212,7 +221,7 @@ int main(int argc, char *argv[]) if (daemon(0, 0)) exit(EXIT_FAILURE); - handle_ras_events(args.record_events); + handle_ras_events(args.record_events, args.enable_ipmitool); return 0; } --- /dev/null +++ b/unified-sel.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2023, Meta Platforms Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ + +#include +#include +#include +#include +#include "ras-record.h" +#include "ras-logger.h" +#include "ras-report.h" +#include "unified-sel.h" + +/* CPU Root Port Error ID corresponding to each status bit set */ +static const char *cor_error_ids[32] = { + /* Correctable errors */ + [0] = "0x00", /* Receiver Error */ + [6] = "0x01", /* Bad TLP */ + [7] = "0x02", /* Bad DLLP */ + [8] = "0x04", /* RELAY_NUM Rollover */ + [12] = "0x03", /* Replay Timer Timeout */ + [13] = "0x05", /* Advisory Non-Fatal */ + [14] = "0x06", /* Corrected Internal */ + [15] = "0x07", /* Header Log Overflow */ +}; + +static int verify_id_log_sel(uint64_t status, + const char **idarray, + unsigned bus, + unsigned dev_fn) +{ + int i; + char openbmc_ipmi_add_sel[105]; + + /* + * Get PCIe AER error source bus/dev/fn and save it to the BMC SEL + * as a OpenBMC unified SEL record type. + * The IPMI command and record fields are defined in IPMI Specification v2.0 (IPMI Spec) + * ipmitool raw 0x0a 0x44 is "Add SEL Entry Command" defined in IPMI spec chapter 31.6 + * The 16 byte that follow form the SEL Record + * defined in IPMI spec chapter 32.1 "SEL Event Records" + * Byte 1~2 are Record ID = 0x00 0x00, unused + * Byte 3 is Record Type = 0xFB, OEM non-timestamped record type for OpenBMC unified SEL + * Byte 4~16 are OEM defined + * Byte 11: + * Byte11[7:3] Device# + * Byte11[2:0] Function# + * Byte 12: Bus number + * Byte 13-15: Reserved + * Byte 16: ID of the error detected on the PCle device that triggered this SEL record + */ + + /* Potentially all error status bits could be set for a given PCIe device. + * Therefore, iterate over all 32 bits each of cor and uncor errors + */ + for (i = 0; i < 32; i++) { + if ((status & (1 << i)) && idarray[i]) { + sprintf(openbmc_ipmi_add_sel, + "ipmitool raw 0x0a 0x44 0x00 0x00 0xFB 0x20 0x00 0x00 0x00 0x00 0x01 0x00 0x%02x 0x%02x 0x01 0x00 0xff %s", + dev_fn, bus, idarray[i]); + if (system(openbmc_ipmi_add_sel) != 0) + return -1; + } + } + return 0; +} + +int openbmc_unified_sel_log(uint64_t severity, const char *dev_name, uint64_t status) +{ + int bus, dev, dev_fn, fn; + + sscanf(dev_name, "%*x:%x:%x.%x", &bus, &dev, &fn); + dev_fn = (((dev & 0x1f) << 3) | (fn & 0x7)); + + /* Use the appropriate correctable error status ID + * for a given severity level + * */ + if (severity == HW_EVENT_AER_CORRECTED) { + if (verify_id_log_sel(status, cor_error_ids, bus, dev_fn) < 0) + return -1; + } + return 0; +} --- /dev/null +++ b/unified-sel.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023, Meta Platforms Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ + + +#ifndef _UNIFIED_SEL_H +#define _UNIFIED_SEL_H + +int openbmc_unified_sel_log(uint64_t severity, const char *dev_name, uint64_t status); + +#endif